* On elisp running native
@ 2019-11-28 6:54 Andrea Corallo
2019-11-28 11:31 ` Óscar Fuentes
` (4 more replies)
0 siblings, 5 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-11-28 6:54 UTC (permalink / raw)
To: emacs-devel
[-- Attachment #1: Type: text/plain, Size: 777 bytes --]
Hi all. I'd like share the investigation I've been working on to give
Emacs the ability to compile and run elisp as native code.
Despite being in very early stage it's functional and shows some quite
interesting potential in terms of performance benefit. I then thought
was at a good stage to be shared and discussed.
I wrote a small page to describe what I did and how it works plus the
current status not to have the information spread in various mails.
http://akrl.sdf.org/gccemacs.html
The repository is hosted at the following link and the current patch
is attached.
https://gitlab.com/koral/gccemacs
In case this is of interest I'd love to see it as feature branch to
better collect feedback and work on its evolution.
Best Regards
Andrea
--
akrl@sdf.org
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: gccemacs.patch --]
[-- Type: text/x-patch, Size: 220672 bytes --]
diff --git a/configure.ac b/configure.ac
index 3b6a2a6..1621844 100644
--- a/configure.ac
+++ b/configure.ac
@@ -463,6 +463,7 @@ AC_DEFUN
OPTION_DEFAULT_ON([zlib],[don't compile with zlib decompression support])
OPTION_DEFAULT_ON([modules],[compile with dynamic modules support])
OPTION_DEFAULT_ON([threads],[don't compile with elisp threading support])
+OPTION_DEFAULT_ON([nativecomp],[don't compile with emacs lisp native compiler support])
AC_ARG_WITH([file-notification],[AS_HELP_STRING([--with-file-notification=LIB],
[use a file notification library (LIB one of: yes, inotify, kqueue, gfile, w32, no)])],
@@ -3724,6 +3725,33 @@ AC_DEFUN
emacs_major_version="${PACKAGE_VERSION%%.*}"
AC_SUBST(emacs_major_version)
+### Emacs Lisp native compiler support
+HAVE_NATIVE_COMP=no
+LIBGCCJIT_LIB=
+COMP_OBJ=
+if test "${with_nativecomp}" != "no"; then
+ AC_CHECK_HEADER([libgccjit.h], [HAVE_NATIVE_COMP=yes])
+ if test "${HAVE_NATIVE_COMP}" = "yes"; then
+ LIBGCCJIT_LIB="-lgccjit -ldl"
+ if test "${HAVE_MODULES}" = yes; then
+ COMP_OBJ="comp.o"
+ else
+ COMP_OBJ="dynlib.o comp.o"
+ fi
+ AC_DEFINE(HAVE_NATIVE_COMP, 1, [Define to 1 if you have the libgccjit library (-lgccjit).])
+ AC_DEFINE_UNQUOTED(NATIVE_ELISP_SUFFIX, ".eln",
+ [System extension for native compiled elisp])
+ else
+ AC_MSG_ERROR([elisp native compiler requested but libgccjit not found.
+If you are sure you want Emacs compiled without elisp native compiler, pass
+ --without-nativecomp
+to configure.])
+ fi
+fi
+AC_SUBST(LIBGCCJIT_LIB)
+AC_SUBST(COMP_OBJ)
+
+
### Use -lpng if available, unless '--with-png=no'.
HAVE_PNG=no
LIBPNG=
@@ -5702,6 +5730,7 @@ AC_DEFUN
Does Emacs support the portable dumper? ${with_pdumper}
Does Emacs support legacy unexec dumping? ${with_unexec}
Which dumping strategy does Emacs use? ${with_dumping}
+ Does Emacs have native lisp compiler? ${HAVE_NATIVE_COMP}
"])
if test -n "${EMACSDATA}"; then
diff --git a/lisp/emacs-lisp/bytecomp.el b/lisp/emacs-lisp/bytecomp.el
index 905d99a..9efa2c6 100644
--- a/lisp/emacs-lisp/bytecomp.el
+++ b/lisp/emacs-lisp/bytecomp.el
@@ -562,6 +562,21 @@ byte-compile-output
(defvar byte-compile-depth 0 "Current depth of execution stack.")
(defvar byte-compile-maxdepth 0 "Maximum depth of execution stack.")
+;; These are use by comp.el to spill data out of here
+(cl-defstruct byte-to-native-function
+ "Named or anonymous function defined a top level."
+ name data)
+(cl-defstruct byte-to-native-top-level
+ "All other top level forms."
+ form)
+(defvar byte-native-compiling nil
+ "t while native compiling.")
+(defvar byte-to-native-lap nil
+ "A-list to accumulate LAP.
+Each pair is (NAME . LAP)")
+(defvar byte-to-native-top-level-forms nil
+ "List of top level forms.")
+
\f
;;; The byte codes; this information is duplicated in bytecomp.c
@@ -2017,7 +2032,9 @@ byte-compile-file
;; emacs-lisp files in the build tree are
;; recompiled). Previously this was accomplished by
;; deleting target-file before writing it.
- (rename-file tempfile target-file t))
+ (if byte-native-compiling
+ (delete-file tempfile)
+ (rename-file tempfile target-file t)))
(or noninteractive (message "Wrote %s" target-file)))
;; This is just to give a better error message than write-region
(let ((exists (file-exists-p target-file)))
@@ -2234,6 +2251,10 @@ byte-compile-output-file-form
;; defalias calls are output directly by byte-compile-file-form-defmumble;
;; it does not pay to first build the defalias in defmumble and then parse
;; it here.
+ (when byte-native-compiling
+ ;; Spill output for the native compiler here
+ (push (make-byte-to-native-top-level :form form)
+ byte-to-native-top-level-forms))
(let ((print-escape-newlines t)
(print-length nil)
(print-level nil)
@@ -2367,7 +2388,8 @@ byte-compile-keep-pending
(defun byte-compile-flush-pending ()
(if byte-compile-output
- (let ((form (byte-compile-out-toplevel t 'file)))
+ (let* ((byte-compile-current-form nil)
+ (form (byte-compile-out-toplevel t 'file)))
(cond ((eq (car-safe form) 'progn)
(mapc 'byte-compile-output-file-form (cdr form)))
(form
@@ -2687,6 +2709,16 @@ byte-compile-file-form-defmumble
;; If there's no doc string, provide -1 as the "doc string
;; index" so that no element will be treated as a doc string.
(if (not (stringp (documentation code t))) -1 4)))
+ (when byte-native-compiling
+ ;; Spill output for the native compiler here.
+ (push (if macro
+ (make-byte-to-native-top-level
+ :form `(defalias ',name '(macro . ,code) nil))
+ (if (commandp code)
+ (make-byte-to-native-top-level ;FIXME compile interactive functions.
+ :form `(defalias ',name ,code))
+ (make-byte-to-native-function :name name :data code)))
+ byte-to-native-top-level-forms))
;; Output the form by hand, that's much simpler than having
;; b-c-output-file-form analyze the defalias.
(byte-compile-output-docform
@@ -3111,9 +3143,14 @@ byte-compile-out-toplevel
(not (delq nil (mapcar 'consp (cdr (car body))))))))
(setq rest (cdr rest)))
rest))
- (let ((byte-compile-vector (byte-compile-constants-vector)))
- (list 'byte-code (byte-compile-lapcode byte-compile-output)
- byte-compile-vector byte-compile-maxdepth)))
+ (let* ((byte-compile-vector (byte-compile-constants-vector))
+ (out (list 'byte-code (byte-compile-lapcode byte-compile-output)
+ byte-compile-vector byte-compile-maxdepth)))
+ (when byte-native-compiling
+ ;; Spill LAP for the native compiler here
+ (push (cons byte-compile-current-form byte-compile-output)
+ byte-to-native-lap))
+ out))
;; it's a trivial function
((cdr body) (cons 'progn (nreverse body)))
((car body)))))
diff --git a/lisp/emacs-lisp/comp.el b/lisp/emacs-lisp/comp.el
new file mode 100644
index 0000000..273bda8
--- /dev/null
+++ b/lisp/emacs-lisp/comp.el
@@ -0,0 +1,1917 @@
+;;; comp.el --- compilation of Lisp code into native code -*- lexical-binding: t -*-
+
+;; Author: Andrea Corallo <akrl@sdf.com>
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; Keywords: lisp
+;; Package: emacs
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;; This code is an attempt to make the pig fly.
+;; Or, to put it another way to make a 911 out of a turbocharged VW Bug.
+
+;;; Code:
+
+(require 'bytecomp)
+(require 'gv)
+(require 'cl-lib)
+(require 'cl-extra)
+(require 'subr-x)
+
+(defgroup comp nil
+ "Emacs Lisp native compiler."
+ :group 'lisp)
+
+(defcustom comp-speed 0
+ "Compiler optimization level. From 0 to 3.
+- 0 no optimizations are performed, compile time is favored.
+- 1 lite optimizations.
+- 2 heavy optimizations.
+- 3 max optimization level, to be used only when necessary.
+ Warning: the compiler is free to perform dangerous optimizations."
+ :type 'number
+ :group 'comp)
+
+(defcustom comp-debug 0
+ "Compiler debug level. From 0 to 3.
+This intended for debugging the compiler itself.
+- 0 no debug facility.
+ This is the recommended value unless you are debugging the compiler itself.
+- 1 emit debug symbols and dump pseudo C code.
+- 2 dump gcc passes and libgccjit log file.
+- 3 dump libgccjit reproducers."
+ :type 'number
+ :group 'comp)
+
+(defcustom comp-verbose 0
+ "Compiler verbosity. From 0 to 3.
+This intended for debugging the compiler itself.
+- 0 no logging.
+- 1 final limple is logged.
+- 2 LAP and final limple and some pass info are logged.
+- 3 max verbosity."
+ :type 'number
+ :group 'comp)
+
+(defcustom comp-always-compile nil
+ "Unconditionally (re-)compile all files."
+ :type 'boolean
+ :group 'comp)
+
+(defconst comp-log-buffer-name "*Native-compile-Log*"
+ "Name of the native-compiler log buffer.")
+
+(defconst comp-async-buffer-name "*Async-native-compile-log*"
+ "Name of the async compilation buffer log.")
+
+(defvar comp-native-compiling nil
+ "This gets bound to t while native compilation.
+Can be used by code that wants to expand differently in this case.")
+
+(defvar comp-pass nil
+ "Every pass has the right to bind what it likes here.")
+
+(defconst comp-passes '(comp-spill-lap
+ comp-limplify
+ comp-ssa
+ comp-propagate
+ comp-call-optim
+ comp-propagate
+ comp-dead-code
+ comp-final)
+ "Passes to be executed in order.")
+
+(defconst comp-known-ret-types '((cons . cons)
+ (1+ . number)
+ (1- . number)
+ (+ . number)
+ (- . number)
+ (* . number)
+ (/ . number)
+ (% . number)
+ ;; Type hints
+ (comp-hint-fixnum . fixnum)
+ (comp-hint-cons . cons))
+ "Alist used for type propagation.")
+
+(defconst comp-type-hints '(comp-hint-fixnum
+ comp-hint-cons)
+ "List of fake functions used to give compiler hints.")
+
+(defconst comp-limple-sets '(set
+ setimm
+ set-par-to-local
+ set-args-to-local
+ set-rest-args-to-local)
+ "Limple set operators.")
+
+(defconst comp-limple-assignments `(fetch-handler
+ ,@comp-limple-sets)
+ "Limple operators that clobbers the first m-var argument.")
+
+(defconst comp-limple-calls '(call
+ callref
+ direct-call
+ direct-callref)
+ "Limple operators use to call subrs.")
+
+(eval-when-compile
+ (defconst comp-op-stack-info
+ (cl-loop with h = (make-hash-table)
+ for k across byte-code-vector
+ for v across byte-stack+-info
+ when k
+ do (puthash k v h)
+ finally return h)
+ "Hash table lap-op -> stack adjustment."))
+
+(cl-defstruct comp-ctxt
+ "Lisp side of the compiler context."
+ (output nil :type string
+ :documentation "Target output file-name for the compilation.")
+ (top-level-forms () :type list
+ :documentation "List of spilled top level forms.")
+ (funcs-h (make-hash-table) :type hash-table
+ :documentation "lisp-func-name -> comp-func.
+This is to build the prev field.")
+ (data-relocs-l () :type list
+ :documentation "Constant objects used by functions.")
+ (data-relocs-idx (make-hash-table :test #'equal) :type hash-table
+ :documentation "Obj -> position into data-relocs.")
+ (func-relocs-l () :type list
+ :documentation "Native functions imported.")
+ (func-relocs-idx (make-hash-table :test #'equal) :type hash-table
+ :documentation "Obj -> position into func-relocs."))
+
+(cl-defstruct comp-args-base
+ (min nil :type number
+ :documentation "Minimum number of arguments allowed."))
+
+(cl-defstruct (comp-args (:include comp-args-base))
+ (max nil :type number
+ :documentation "Maximum number of arguments allowed.
+To be used when ncall-conv is nil."))
+
+(cl-defstruct (comp-nargs (:include comp-args-base))
+ "Describe args when the function signature is of kind:
+(ptrdiff_t nargs, Lisp_Object *args)."
+ (nonrest nil :type number
+ :documentation "Number of non rest arguments.")
+ (rest nil :type boolean
+ :documentation "t if rest argument is present."))
+
+(cl-defstruct (comp-block (:copier nil)
+ (:constructor make--comp-block
+ (addr sp name))) ; Positional
+ "A basic block."
+ (name nil :type symbol)
+ ;; These two slots are used during limplification.
+ (sp nil :type number
+ :documentation "When non nil indicates the sp value while entering
+into it.")
+ (addr nil :type number
+ :documentation "Start block LAP address.")
+ (insns () :type list
+ :documentation "List of instructions.")
+ (closed nil :type boolean
+ :documentation "t if closed.")
+ ;; All the followings are for SSA and CGF analysis.
+ (in-edges () :type list
+ :documentation "List of incoming edges.")
+ (out-edges () :type list
+ :documentation "List of out-coming edges.")
+ (dom nil :type comp-block
+ :documentation "Immediate dominator.")
+ (df (make-hash-table) :type hash-table
+ :documentation "Dominance frontier set. Block-name -> block")
+ (post-num nil :type number
+ :documentation "Post order number.")
+ (final-frame nil :type vector
+ :documentation "This is a copy of the frame when leaving the block.
+Is in use to help the SSA rename pass."))
+
+(cl-defstruct (comp-edge (:copier nil) (:constructor make--comp-edge))
+ "An edge connecting two basic blocks."
+ (src nil :type comp-block)
+ (dst nil :type comp-block)
+ (number nil :type number
+ :documentation "The index number corresponding to this edge in the
+ edge vector."))
+
+(defun comp-block-preds (basic-block)
+ "Given BASIC-BLOCK return the list of its predecessors."
+ (mapcar #'comp-edge-src (comp-block-in-edges basic-block)))
+
+(defun comp-gen-counter ()
+ "Return a sequential number generator."
+ (let ((n -1))
+ (lambda ()
+ (cl-incf n))))
+
+(cl-defstruct (comp-func (:copier nil))
+ "LIMPLE representation of a function."
+ (name nil :type symbol
+ :documentation "Function symbol name.")
+ (c-name nil :type string
+ :documentation "The function name in the native world.")
+ (byte-func nil
+ :documentation "Byte compiled version.")
+ (doc nil :type string
+ :documentation "Doc string.")
+ (lap () :type list
+ :documentation "LAP assembly representation.")
+ (args nil :type comp-args-base)
+ (frame-size nil :type number)
+ (blocks (make-hash-table) :type hash-table
+ :documentation "Key is the basic block symbol value is a comp-block
+structure.")
+ (lap-block (make-hash-table :test #'equal) :type hash-table
+ :documentation "LAP lable -> LIMPLE basic block name.")
+ (edges () :type list
+ :documentation "List of edges connecting basic blocks.")
+ (block-cnt-gen (funcall #'comp-gen-counter) :type function
+ :documentation "Generates block numbers.")
+ (edge-cnt-gen (funcall #'comp-gen-counter) :type function
+ :documentation "Generates edges numbers.")
+ (ssa-cnt-gen (funcall #'comp-gen-counter) :type function
+ :documentation "Counter to create ssa limple vars."))
+
+(defun comp-func-reset-generators (func)
+ "Reset unique id generators for FUNC."
+ (setf (comp-func-edge-cnt-gen func) (comp-gen-counter)
+ (comp-func-ssa-cnt-gen func) (comp-gen-counter)))
+
+(cl-defstruct (comp-mvar (:constructor make--comp-mvar))
+ "A meta-variable being a slot in the meta-stack."
+ (slot nil :type fixnum
+ :documentation "Slot number.
+-1 is a special value and indicates the scratch slot.")
+ (id nil :type (or null number)
+ :documentation "SSA number when in SSA form.")
+ (const-vld nil :type boolean
+ :documentation "Valid signal for the following slot.")
+ (constant nil
+ :documentation "When const-vld non nil this is used for holding
+ a value known at compile time.")
+ (type nil
+ :documentation "When non nil indicates the type when known at compile
+ time.")
+ (ref nil :type boolean
+ :documentation "When t the m-var is involved in a call where is passed by
+ reference."))
+
+;; Special vars used by some passes
+(defvar comp-func)
+
+\f
+
+(defun comp-set-op-p (op)
+ "Assignment predicate for OP."
+ (cl-find op comp-limple-sets))
+
+(defun comp-assign-op-p (op)
+ "Assignment predicate for OP."
+ (cl-find op comp-limple-assignments))
+
+(defun comp-limple-insn-call-p (insn)
+ "Limple INSN call predicate."
+ (when (member (car-safe insn) comp-limple-calls)
+ t))
+
+(defun comp-type-hint-p (func)
+ "Type hint predicate for function name FUNC."
+ (member func comp-type-hints))
+
+(defun comp-add-const-to-relocs (obj)
+ "Keep track of OBJ into the ctxt relocations.
+The corresponding index is returned."
+ (let ((data-relocs-idx (comp-ctxt-data-relocs-idx comp-ctxt)))
+ (if-let ((idx (gethash obj data-relocs-idx)))
+ idx
+ (push obj (comp-ctxt-data-relocs-l comp-ctxt))
+ (puthash obj (hash-table-count data-relocs-idx) data-relocs-idx))))
+
+(defun comp-add-subr-to-relocs (subr-name)
+ "Keep track of SUBR-NAME into the ctxt relocations.
+The corresponding index is returned."
+ (let ((func-relocs-idx (comp-ctxt-func-relocs-idx comp-ctxt)))
+ (if-let ((idx (gethash subr-name func-relocs-idx)))
+ idx
+ (push subr-name (comp-ctxt-func-relocs-l comp-ctxt))
+ (puthash subr-name (hash-table-count func-relocs-idx) func-relocs-idx))))
+
+(defmacro comp-within-log-buff (&rest body)
+ "Execute BODY while at the end the log-buffer.
+BODY is evaluate only if `comp-verbose' is > 0."
+ (declare (debug (form body))
+ (indent defun))
+ `(when (> comp-verbose 0)
+ (with-current-buffer (get-buffer-create comp-log-buffer-name)
+ (setf buffer-read-only t)
+ (let ((inhibit-read-only t))
+ (goto-char (point-max))
+ ,@body))))
+
+(defun comp-log (data verbosity)
+ "Log DATA given VERBOSITY."
+ (when (>= comp-verbose verbosity)
+ (if noninteractive
+ (if (atom data)
+ (message "%s" data)
+ (mapc (lambda (x)
+ (message "%s"(prin1-to-string x)))
+ data))
+ (comp-within-log-buff
+ (if (and data (atom data))
+ (insert data)
+ (mapc (lambda (x)
+ (insert (prin1-to-string x) "\n"))
+ data)
+ (insert "\n"))))))
+
+(defun comp-log-func (func verbosity)
+ "Log function FUNC.
+VERBOSITY is a number between 0 and 3."
+ (when (>= comp-verbose verbosity)
+ (comp-log (format "\nFunction: %s\n" (comp-func-name func)) verbosity)
+ (cl-loop for block-name being each hash-keys of (comp-func-blocks func)
+ using (hash-value bb)
+ do (comp-log (concat "<" (symbol-name block-name) ">") verbosity)
+ (comp-log (comp-block-insns bb) verbosity))))
+
+(defun comp-log-edges (func)
+ "Log edges in FUNC."
+ (let ((edges (comp-func-edges func)))
+ (comp-log (format "\nEdges in function: %s\n"
+ (comp-func-name func))
+ 2)
+ (mapc (lambda (e)
+ (comp-log (format "n: %d src: %s dst: %s\n"
+ (comp-edge-number e)
+ (comp-block-name (comp-edge-src e))
+ (comp-block-name (comp-edge-dst e)))
+ 2))
+ edges)))
+
+\f
+;;; spill-lap pass specific code.
+
+(defun comp-c-func-name (symbol prefix)
+ "Given SYMBOL return a name suitable for the native code.
+Put PREFIX in front of it."
+ ;; Unfortunatelly not all symbol names are valid as C function names...
+ ;; Nassi's algorithm here:
+ (let* ((orig-name (symbol-name symbol))
+ (crypted (cl-loop with str = (make-string (* 2 (length orig-name)) 0)
+ for j from 0 by 2
+ for i across orig-name
+ for byte = (format "%x" i)
+ do (aset str j (aref byte 0))
+ (aset str (1+ j) (aref byte 1))
+ finally return str))
+ (human-readable (replace-regexp-in-string
+ "-" "_" orig-name))
+ (human-readable (replace-regexp-in-string
+ (rx (not (any "0-9a-z_"))) "" human-readable)))
+ (concat prefix crypted "_" human-readable)))
+
+(defun comp-decrypt-lambda-list (x)
+ "Decript lambda list X."
+ (unless (fixnump x)
+ (signal 'native-compiler-error
+ "can't native compile a non lexical scoped function"))
+ (let ((rest (not (= (logand x 128) 0)))
+ (mandatory (logand x 127))
+ (nonrest (ash x -8)))
+ (if (and (null rest)
+ (< nonrest 9)) ;; SUBR_MAX_ARGS
+ (make-comp-args :min mandatory
+ :max nonrest)
+ (make-comp-nargs :min mandatory
+ :nonrest nonrest
+ :rest rest))))
+
+(defsubst comp-byte-frame-size (byte-compiled-func)
+ "Given BYTE-COMPILED-FUNC return the frame size to be allocated."
+ (aref byte-compiled-func 3))
+
+(cl-defgeneric comp-spill-lap-function (input)
+ "Byte compile INPUT and spill lap for further stages.")
+
+(cl-defgeneric comp-spill-lap-function ((function-name symbol))
+ "Byte compile FUNCTION-NAME spilling data from the byte compiler."
+ (let* ((f (symbol-function function-name))
+ (func (make-comp-func :name function-name
+ :c-name (comp-c-func-name function-name"F"))))
+ (when (byte-code-function-p f)
+ (signal 'native-compiler-error
+ "can't native compile an already bytecompiled function"))
+ (setf (comp-func-byte-func func)
+ (byte-compile (comp-func-name func)))
+ (let ((lap (alist-get nil byte-to-native-lap)))
+ (cl-assert lap)
+ (comp-log lap 2)
+ (let ((lambda-list (aref (comp-func-byte-func func) 0)))
+ (setf (comp-func-args func)
+ (comp-decrypt-lambda-list lambda-list)
+ (comp-func-lap func)
+ lap
+ (comp-func-frame-size func)
+ (comp-byte-frame-size (comp-func-byte-func func))))
+ (setf (comp-ctxt-top-level-forms comp-ctxt)
+ (list (make-byte-to-native-function :name function-name)))
+ (list func))))
+
+(cl-defgeneric comp-spill-lap-function ((filename string))
+ "Byte compile FILENAME spilling data from the byte compiler."
+ (byte-compile-file filename)
+ (unless byte-to-native-top-level-forms
+ (signal 'native-compiler-error "empty byte compiler output"))
+ (setf (comp-ctxt-top-level-forms comp-ctxt) (reverse byte-to-native-top-level-forms))
+ (cl-loop
+ for f in (cl-loop for x in byte-to-native-top-level-forms ; All non anonymous.
+ when (and (byte-to-native-function-p x)
+ (byte-to-native-function-name x))
+ collect x)
+ for name = (byte-to-native-function-name f)
+ for data = (byte-to-native-function-data f)
+ for doc = (when (>= (length data) 5) (aref data 4))
+ for lap = (alist-get name byte-to-native-lap)
+ for lambda-list = (aref data 0)
+ for func = (make-comp-func :name name
+ :byte-func data
+ :doc doc
+ :c-name (comp-c-func-name name "F")
+ :args (comp-decrypt-lambda-list lambda-list)
+ :lap lap
+ :frame-size (comp-byte-frame-size data))
+ do (comp-log (format "Function %s:\n" name) 1)
+ (comp-log lap 1)
+ collect func))
+
+(defun comp-spill-lap (input)
+ "Byte compile and spill the LAP representation for INPUT.
+If INPUT is a symbol this is the function-name to be compiled.
+If INPUT is a string this is the file path to be compiled."
+ (let ((byte-native-compiling t)
+ (byte-to-native-lap ())
+ (byte-to-native-top-level-forms ()))
+ (comp-spill-lap-function input)))
+
+\f
+;;; Limplification pass specific code.
+
+(cl-defstruct (comp-limplify (:copier nil))
+ "Support structure used during function limplification."
+ (frame nil :type vector
+ :documentation "Meta-stack used to flat LAP.")
+ (curr-block nil :type comp-block
+ :documentation "Current block being limplified.")
+ (sp -1 :type number
+ :documentation "Current stack pointer while walking LAP.
+Points to the next slot to be filled.")
+ (pc 0 :type number
+ :documentation "Current program counter while walking LAP.")
+ (label-to-addr nil :type hash-table
+ :documentation "LAP hash table -> address.")
+ (pending-blocks () :type list
+ :documentation "List of blocks waiting for limplification."))
+
+(defconst comp-lap-eob-ops
+ '(byte-goto byte-goto-if-nil byte-goto-if-not-nil byte-goto-if-nil-else-pop
+ byte-goto-if-not-nil-else-pop byte-return byte-pushcatch
+ byte-switch byte-pushconditioncase)
+ "LAP end of basic blocks op codes.")
+
+(defsubst comp-lap-eob-p (inst)
+ "Return t if INST closes the current basic blocks, nil otherwise."
+ (when (member (car inst) comp-lap-eob-ops)
+ t))
+
+(defsubst comp-lap-fall-through-p (inst)
+ "Return t if INST fall through, nil otherwise."
+ (when (not (member (car inst) '(byte-goto byte-return)))
+ t))
+
+(defsubst comp-sp ()
+ "Current stack pointer."
+ (comp-limplify-sp comp-pass))
+(gv-define-setter comp-sp (value)
+ `(setf (comp-limplify-sp comp-pass) ,value))
+
+(defmacro comp-with-sp (sp &rest body)
+ "Execute BODY setting the stack pointer to SP.
+Restore the original value afterwards."
+ (declare (debug (form body))
+ (indent defun))
+ (let ((sym (gensym)))
+ `(let ((,sym (comp-sp)))
+ (setf (comp-sp) ,sp)
+ (progn ,@body)
+ (setf (comp-sp) ,sym))))
+
+(defsubst comp-slot-n (n)
+ "Slot N into the meta-stack."
+ (aref (comp-limplify-frame comp-pass) n))
+
+(defsubst comp-slot ()
+ "Current slot into the meta-stack pointed by sp."
+ (comp-slot-n (comp-sp)))
+
+(defsubst comp-slot+1 ()
+ "Slot into the meta-stack pointed by sp + 1."
+ (comp-slot-n (1+ (comp-sp))))
+
+(defsubst comp-label-to-addr (label)
+ "Find the address of LABEL."
+ (or (gethash label (comp-limplify-label-to-addr comp-pass))
+ (signal 'native-ice (list "label not found" label))))
+
+(defsubst comp-mark-curr-bb-closed ()
+ "Mark the current basic block as closed."
+ (setf (comp-block-closed (comp-limplify-curr-block comp-pass)) t))
+
+(defun comp-bb-maybe-add (lap-addr &optional sp)
+ "If necessary create a pending basic block for LAP-ADDR with stack depth SP.
+The basic block is returned regardless it was already declared or not."
+ (let ((bb (or (cl-loop ; See if the block was already liplified.
+ for bb being the hash-value in (comp-func-blocks comp-func)
+ when (equal (comp-block-addr bb) lap-addr)
+ return bb)
+ (cl-find-if (lambda (bb) ; Look within the pendings blocks.
+ (= (comp-block-addr bb) lap-addr))
+ (comp-limplify-pending-blocks comp-pass)))))
+ (if bb
+ (progn
+ (unless (or (null sp) (= sp (comp-block-sp bb)))
+ (signal 'native-ice (list "incoherent stack pointers"
+ sp (comp-block-sp bb))))
+ bb)
+ (car (push (make--comp-block lap-addr sp (comp-new-block-sym))
+ (comp-limplify-pending-blocks comp-pass))))))
+
+(defun comp-call (func &rest args)
+ "Emit a call for function FUNC with ARGS."
+ (comp-add-subr-to-relocs func)
+ `(call ,func ,@args))
+
+(defun comp-callref (func nargs stack-off)
+ "Emit a call using narg abi for FUNC.
+NARGS is the number of arguments.
+STACK-OFF is the index of the first slot frame involved."
+ (comp-add-subr-to-relocs func)
+ `(callref ,func ,@(cl-loop repeat nargs
+ for sp from stack-off
+ collect (comp-slot-n sp))))
+
+(cl-defun make-comp-mvar (&key slot (constant nil const-vld) type)
+ (when const-vld
+ (comp-add-const-to-relocs constant))
+ (make--comp-mvar :slot slot :const-vld const-vld :constant constant
+ :type type))
+
+(defun comp-new-frame (size &optional ssa)
+ "Return a clean frame of meta variables of size SIZE.
+If SSA non nil populate it of m-var in ssa form."
+ (cl-loop with v = (make-vector size nil)
+ for i below size
+ for mvar = (if ssa
+ (make-comp-ssa-mvar :slot i)
+ (make-comp-mvar :slot i))
+ do (aset v i mvar)
+ finally return v))
+
+(defsubst comp-emit (insn)
+ "Emit INSN into basic block BB."
+ (let ((bb (comp-limplify-curr-block comp-pass)))
+ (cl-assert (not (comp-block-closed bb)))
+ (push insn (comp-block-insns bb))))
+
+(defsubst comp-emit-set-call (call)
+ "Emit CALL assigning the result the the current slot frame.
+If the callee function is known to have a return type propagate it."
+ (cl-assert call)
+ (comp-emit (list 'set (comp-slot) call)))
+
+(defun comp-copy-slot (src-n &optional dst-n)
+ "Set slot number DST-N to slot number SRC-N as source.
+If DST-N is specified use it otherwise assume it to be the current slot."
+ (comp-with-sp (or dst-n (comp-sp))
+ (let ((src-slot (comp-slot-n src-n)))
+ (cl-assert src-slot)
+ (comp-emit `(set ,(comp-slot) ,src-slot)))))
+
+(defsubst comp-emit-annotation (str)
+ "Emit annotation STR."
+ (comp-emit `(comment ,str)))
+
+(defun comp-emit-set-const (val)
+ "Set constant VAL to current slot."
+ (let ((rel-idx (comp-add-const-to-relocs val)))
+ (cl-assert (numberp rel-idx))
+ (comp-emit `(setimm ,(comp-slot) ,rel-idx ,val))))
+
+(defun comp-make-curr-block (block-name entry-sp &optional addr)
+ "Create a basic block with BLOCK-NAME and set it as current block.
+ENTRY-SP is the sp value when entering.
+The block is added to the current function.
+The block is returned."
+ (let ((bb (make--comp-block addr entry-sp block-name)))
+ (setf (comp-limplify-curr-block comp-pass) bb
+ (comp-limplify-pc comp-pass) addr
+ (comp-limplify-sp comp-pass) (comp-block-sp bb))
+ (puthash (comp-block-name bb) bb (comp-func-blocks comp-func))
+ bb))
+
+(defun comp-emit-uncond-jump (lap-label)
+ "Emit an unconditional branch to LAP-LABEL."
+ (cl-destructuring-bind (label-num . stack-depth) lap-label
+ (when stack-depth
+ (cl-assert (= (1- stack-depth) (comp-sp))))
+ (let ((target (comp-bb-maybe-add (comp-label-to-addr label-num)
+ (comp-sp))))
+ (comp-emit `(jump ,(comp-block-name target)))
+ (comp-mark-curr-bb-closed))))
+
+(defun comp-emit-cond-jump (a b target-offset lap-label negated)
+ "Emit a conditional jump to LAP-LABEL when A and B satisfy EQ.
+TARGET-OFFSET is the positive offset on the SP when branching to the target
+block.
+If NEGATED non null negate the tested condition.
+Return value is the fall through block name."
+ (cl-destructuring-bind (label-num . label-sp) lap-label
+ (let* ((bb (comp-block-name (comp-bb-maybe-add (1+ (comp-limplify-pc comp-pass))
+ (comp-sp)))) ; Fall through block.
+ (target-sp (+ target-offset (comp-sp)))
+ (target (comp-block-name (comp-bb-maybe-add (comp-label-to-addr label-num)
+ target-sp))))
+ (when label-sp
+ (cl-assert (= (1- label-sp) (+ target-offset (comp-sp)))))
+ (comp-emit (if negated
+ (list 'cond-jump a b target bb)
+ (list 'cond-jump a b bb target)))
+ (comp-mark-curr-bb-closed)
+ bb)))
+
+(defun comp-emit-handler (lap-label handler-type)
+ "Emit a non local exit handler to LAP-LABEL of type HANDLER-TYPE."
+ (cl-destructuring-bind (label-num . label-sp) lap-label
+ (cl-assert (= (- label-sp 2) (comp-sp)))
+ (let* ((guarded-bb (comp-bb-maybe-add (1+ (comp-limplify-pc comp-pass))
+ (comp-sp)))
+ (handler-bb (comp-bb-maybe-add (comp-label-to-addr label-num)
+ (1+ (comp-sp))))
+ (pop-bb (make--comp-block nil (comp-sp) (comp-new-block-sym))))
+ (comp-emit (list 'push-handler
+ handler-type
+ (comp-slot+1)
+ (comp-block-name pop-bb)
+ (comp-block-name guarded-bb)))
+ (comp-mark-curr-bb-closed)
+ ;; Emit the basic block to pop the handler if we got the non local.
+ (puthash (comp-block-name pop-bb) pop-bb (comp-func-blocks comp-func))
+ (setf (comp-limplify-curr-block comp-pass) pop-bb)
+ (comp-emit `(fetch-handler ,(comp-slot+1)))
+ (comp-emit `(jump ,(comp-block-name handler-bb)))
+ (comp-mark-curr-bb-closed))))
+
+(defun comp-limplify-listn (n)
+ "Limplify list N."
+ (comp-with-sp (+ (comp-sp) n -1)
+ (comp-emit-set-call (comp-call 'cons
+ (comp-slot)
+ (make-comp-mvar :constant nil))))
+ (cl-loop for sp from (+ (comp-sp) n -2) downto (comp-sp)
+ do (comp-with-sp sp
+ (comp-emit-set-call (comp-call 'cons
+ (comp-slot)
+ (comp-slot+1))))))
+
+(defun comp-new-block-sym ()
+ "Return a unique symbol naming the next new basic block."
+ (intern (format "bb_%s" (funcall (comp-func-block-cnt-gen comp-func)))))
+
+(defun comp-fill-label-h ()
+ "Fill label-to-addr hash table for the current function."
+ (setf (comp-limplify-label-to-addr comp-pass) (make-hash-table :test 'eql))
+ (cl-loop for insn in (comp-func-lap comp-func)
+ for addr from 0
+ do (pcase insn
+ (`(TAG ,label . ,_)
+ (puthash label addr (comp-limplify-label-to-addr comp-pass))))))
+
+(defun comp-emit-switch (var last-insn)
+ "Emit a limple for a lap jump table given VAR and LAST-INSN."
+ ;; FIXME this not efficient for big jump tables. We should have a second
+ ;; strategy for this case.
+ (pcase last-insn
+ (`(setimm ,_ ,_ ,jmp-table)
+ (cl-loop
+ for test being each hash-keys of jmp-table
+ using (hash-value target-label)
+ with len = (hash-table-count jmp-table)
+ with test-func = (hash-table-test jmp-table)
+ for n from 1
+ for last = (= n len)
+ for m-test = (make-comp-mvar :constant test)
+ for target-name = (comp-block-name (comp-bb-maybe-add (comp-label-to-addr target-label)
+ (comp-sp)))
+ for ff-bb = (if last
+ (comp-bb-maybe-add (1+ (comp-limplify-pc comp-pass))
+ (comp-sp))
+ (make--comp-block nil
+ (comp-sp)
+ (comp-new-block-sym)))
+ for ff-bb-name = (comp-block-name ff-bb)
+ if (eq test-func 'eq)
+ do (comp-emit (list 'cond-jump var m-test ff-bb-name target-name))
+ else
+ ;; Store the result of the comparison into the scratch slot before
+ ;; emitting the conditional jump.
+ do (comp-emit (list 'set (make-comp-mvar :slot -1)
+ (comp-call test-func var m-test)))
+ (comp-emit (list 'cond-jump
+ (make-comp-mvar :slot -1)
+ (make-comp-mvar :constant nil)
+ target-name ff-bb-name))
+ do (unless last
+ ;; All fall through are artificially created here except the last one.
+ (puthash ff-bb-name ff-bb (comp-func-blocks comp-func))
+ (setf (comp-limplify-curr-block comp-pass) ff-bb))))
+ (_ (signal 'native-ice
+ "missing previous setimm while creating a switch"))))
+
+(defun comp-emit-set-call-subr (subr-name sp-delta)
+ "Emit a call for SUBR-NAME.
+SP-DELTA is the stack adjustment."
+ (let ((subr (symbol-function subr-name))
+ (nargs (1+ (- sp-delta))))
+ (unless (subrp subr)
+ (signal 'native-ice (list "not a subr" subr)))
+ (let* ((arity (subr-arity subr))
+ (minarg (car arity))
+ (maxarg (cdr arity)))
+ (when (eq maxarg 'unevalled)
+ (signal 'native-ice (list "subr contains unevalled args" subr-name)))
+ (if (eq maxarg 'many)
+ ;; callref case.
+ (comp-emit-set-call (comp-callref subr-name nargs (comp-sp)))
+ ;; Normal call.
+ (unless (and (>= maxarg nargs) (<= minarg nargs))
+ (signal 'native-ice
+ (list "incoherent stack adjustment" nargs maxarg minarg)))
+ (let* ((subr-name subr-name)
+ (slots (cl-loop for i from 0 below maxarg
+ collect (comp-slot-n (+ i (comp-sp))))))
+ (comp-emit-set-call (apply #'comp-call (cons subr-name slots))))))))
+
+(eval-when-compile
+ (defun comp-op-to-fun (x)
+ "Given the LAP op strip \"byte-\" to have the subr name."
+ (intern (replace-regexp-in-string "byte-" "" x)))
+
+ (defun comp-body-eff (body op-name sp-delta)
+ "Given the original body BODY compute the effective one.
+When BODY is auto guess function name form the LAP byte-code
+name. Otherwise expect lname fnname."
+ (pcase (car body)
+ ('auto
+ (list `(comp-emit-set-call-subr
+ ',(comp-op-to-fun op-name)
+ ,sp-delta)))
+ ((pred symbolp)
+ (list `(comp-emit-set-call-subr
+ ',(car body)
+ ,sp-delta)))
+ (_ body))))
+
+(defmacro comp-op-case (&rest cases)
+ "Expand CASES into the corresponding `pcase' expansion.
+This is responsible for generating the proper stack adjustment when known and
+the annotation emission."
+ (declare (debug (body))
+ (indent defun))
+ `(pcase op
+ ,@(cl-loop for (op . body) in cases
+ for sp-delta = (gethash op comp-op-stack-info)
+ for op-name = (symbol-name op)
+ if body
+ collect `(',op
+ ;; Log all LAP ops except the TAG one.
+ ,(unless (eq op 'TAG)
+ `(comp-emit-annotation
+ ,(concat "LAP op " op-name)))
+ ;; Emit the stack adjustment if present.
+ ,(when (and sp-delta (not (eq 0 sp-delta)))
+ `(cl-incf (comp-sp) ,sp-delta))
+ ,@(comp-body-eff body op-name sp-delta))
+ else
+ collect `(',op (signal 'native-ice
+ (list "unsupported LAP op" ',op-name))))
+ (_ (signal 'native-ice (list "unexpected LAP op" (symbol-name op))))))
+
+(defun comp-limplify-lap-inst (insn)
+ "Limplify LAP instruction INSN pushing it in the proper basic block."
+ (let ((op (car insn))
+ (arg (if (consp (cdr insn))
+ (cadr insn)
+ (cdr insn))))
+ (comp-op-case
+ (TAG
+ (cl-destructuring-bind (_TAG label-num . label-sp) insn
+ ;; Paranoid?
+ (when label-sp
+ (cl-assert (= (1- label-sp) (comp-limplify-sp comp-pass))))
+ (comp-emit-annotation (format "LAP TAG %d" label-num))))
+ (byte-stack-ref
+ (comp-copy-slot (- (comp-sp) arg 1)))
+ (byte-varref
+ (comp-emit-set-call (comp-call 'symbol-value (make-comp-mvar
+ :constant arg))))
+ (byte-varset
+ (comp-emit (comp-call 'set_internal
+ (make-comp-mvar :constant arg)
+ (comp-slot+1))))
+ (byte-varbind ;; Verify
+ (comp-emit (comp-call 'specbind
+ (make-comp-mvar :constant arg)
+ (comp-slot+1))))
+ (byte-call
+ (cl-incf (comp-sp) (- arg))
+ (comp-emit-set-call (comp-callref 'funcall (1+ arg) (comp-sp))))
+ (byte-unbind
+ (comp-emit (comp-call 'helper_unbind_n
+ (make-comp-mvar :constant arg))))
+ (byte-pophandler
+ (comp-emit '(pop-handler)))
+ (byte-pushconditioncase
+ (comp-emit-handler (cddr insn) 'condition-case))
+ (byte-pushcatch
+ (comp-emit-handler (cddr insn) 'catcher))
+ (byte-nth auto)
+ (byte-symbolp auto)
+ (byte-consp auto)
+ (byte-stringp auto)
+ (byte-listp auto)
+ (byte-eq auto)
+ (byte-memq auto)
+ (byte-not null)
+ (byte-car auto)
+ (byte-cdr auto)
+ (byte-cons auto)
+ (byte-list1
+ (comp-limplify-listn 1))
+ (byte-list2
+ (comp-limplify-listn 2))
+ (byte-list3
+ (comp-limplify-listn 3))
+ (byte-list4
+ (comp-limplify-listn 4))
+ (byte-length auto)
+ (byte-aref auto)
+ (byte-aset auto)
+ (byte-symbol-value auto)
+ (byte-symbol-function auto)
+ (byte-set auto)
+ (byte-fset auto)
+ (byte-get auto)
+ (byte-substring auto)
+ (byte-concat2
+ (comp-emit-set-call (comp-callref 'concat 2 (comp-sp))))
+ (byte-concat3
+ (comp-emit-set-call (comp-callref 'concat 3 (comp-sp))))
+ (byte-concat4
+ (comp-emit-set-call (comp-callref 'concat 4 (comp-sp))))
+ (byte-sub1 1-)
+ (byte-add1 1+)
+ (byte-eqlsign =)
+ (byte-gtr >)
+ (byte-lss <)
+ (byte-leq <=)
+ (byte-geq >=)
+ (byte-diff -)
+ (byte-negate
+ (comp-emit-set-call (comp-call 'negate (comp-slot))))
+ (byte-plus +)
+ (byte-max auto)
+ (byte-min auto)
+ (byte-mult *)
+ (byte-point auto)
+ (byte-goto-char auto)
+ (byte-insert auto)
+ (byte-point-max auto)
+ (byte-point-min auto)
+ (byte-char-after auto)
+ (byte-following-char auto)
+ (byte-preceding-char preceding-char)
+ (byte-current-column auto)
+ (byte-indent-to
+ (comp-emit-set-call (comp-call 'indent-to
+ (comp-slot)
+ (make-comp-mvar :constant nil))))
+ (byte-scan-buffer-OBSOLETE)
+ (byte-eolp auto)
+ (byte-eobp auto)
+ (byte-bolp auto)
+ (byte-bobp auto)
+ (byte-current-buffer auto)
+ (byte-set-buffer auto)
+ (byte-save-current-buffer
+ (comp-emit (comp-call 'record_unwind_current_buffer)))
+ (byte-set-mark-OBSOLETE)
+ (byte-interactive-p-OBSOLETE)
+ (byte-forward-char auto)
+ (byte-forward-word auto)
+ (byte-skip-chars-forward auto)
+ (byte-skip-chars-backward auto)
+ (byte-forward-line auto)
+ (byte-char-syntax auto)
+ (byte-buffer-substring auto)
+ (byte-delete-region auto)
+ (byte-narrow-to-region
+ (comp-emit-set-call (comp-call 'narrow-to-region
+ (comp-slot)
+ (comp-slot+1))))
+ (byte-widen
+ (comp-emit-set-call (comp-call 'widen)))
+ (byte-end-of-line auto)
+ (byte-constant2) ; TODO
+ ;; Branches.
+ (byte-goto
+ (comp-emit-uncond-jump (cddr insn)))
+ (byte-goto-if-nil
+ (comp-emit-cond-jump (comp-slot+1) (make-comp-mvar :constant nil) 0
+ (cddr insn) nil))
+ (byte-goto-if-not-nil
+ (comp-emit-cond-jump (comp-slot+1) (make-comp-mvar :constant nil) 0
+ (cddr insn) t))
+ (byte-goto-if-nil-else-pop
+ (comp-emit-cond-jump (comp-slot+1) (make-comp-mvar :constant nil) 1
+ (cddr insn) nil))
+ (byte-goto-if-not-nil-else-pop
+ (comp-emit-cond-jump (comp-slot+1) (make-comp-mvar :constant nil) 1
+ (cddr insn) t))
+ (byte-return
+ (comp-emit `(return ,(comp-slot+1))))
+ (byte-discard 'pass)
+ (byte-dup
+ (comp-copy-slot (1- (comp-sp))))
+ (byte-save-excursion
+ (comp-emit (comp-call 'record_unwind_protect_excursion)))
+ (byte-save-window-excursion-OBSOLETE)
+ (byte-save-restriction
+ (comp-emit (comp-call 'helper_save_restriction)))
+ (byte-catch) ;; Obsolete
+ (byte-unwind-protect
+ (comp-emit (comp-call 'helper_unwind_protect (comp-slot+1))))
+ (byte-condition-case) ;; Obsolete
+ (byte-temp-output-buffer-setup-OBSOLETE)
+ (byte-temp-output-buffer-show-OBSOLETE)
+ (byte-unbind-all) ;; Obsolete
+ (byte-set-marker auto)
+ (byte-match-beginning auto)
+ (byte-match-end auto)
+ (byte-upcase auto)
+ (byte-downcase auto)
+ (byte-string= string-equal)
+ (byte-string< string-lessp)
+ (byte-equal auto)
+ (byte-nthcdr auto)
+ (byte-elt auto)
+ (byte-member auto)
+ (byte-assq auto)
+ (byte-nreverse auto)
+ (byte-setcar auto)
+ (byte-setcdr auto)
+ (byte-car-safe auto)
+ (byte-cdr-safe auto)
+ (byte-nconc auto)
+ (byte-quo /)
+ (byte-rem %)
+ (byte-numberp auto)
+ (byte-integerp auto)
+ (byte-listN
+ (cl-incf (comp-sp) (- 1 arg))
+ (comp-emit-set-call (comp-callref 'list arg (comp-sp))))
+ (byte-concatN
+ (cl-incf (comp-sp) (- 1 arg))
+ (comp-emit-set-call (comp-callref 'concat arg (comp-sp))))
+ (byte-insertN
+ (cl-incf (comp-sp) (- 1 arg))
+ (comp-emit-set-call (comp-callref 'insert arg (comp-sp))))
+ (byte-stack-set
+ (comp-copy-slot (1+ (comp-sp)) (- (comp-sp) arg -1)))
+ (byte-stack-set2 (cl-assert nil)) ;; TODO
+ (byte-discardN
+ (cl-incf (comp-sp) (- arg)))
+ (byte-switch
+ ;; Assume to follow the emission of a setimm.
+ ;; This is checked into comp-emit-switch.
+ (comp-emit-switch (comp-slot+1)
+ (cl-second (comp-block-insns
+ (comp-limplify-curr-block comp-pass)))))
+ (byte-constant
+ (comp-emit-set-const arg))
+ (byte-discardN-preserve-tos
+ (cl-incf (comp-sp) (- arg))
+ (comp-copy-slot (+ arg (comp-sp)))))))
+
+(defun comp-emit-narg-prologue (minarg nonrest rest)
+ "Emit the prologue for a narg function."
+ (cl-loop for i below minarg
+ do (comp-emit `(set-args-to-local ,(comp-slot-n i)))
+ (comp-emit '(inc-args)))
+ (cl-loop for i from minarg below nonrest
+ for bb = (intern (format "entry_%s" i))
+ for fallback = (intern (format "entry_fallback_%s" i))
+ do (comp-emit `(cond-jump-narg-leq ,i ,bb ,fallback))
+ (comp-make-curr-block bb (comp-sp))
+ (comp-emit `(set-args-to-local ,(comp-slot-n i)))
+ (comp-emit '(inc-args))
+ finally (comp-emit '(jump entry_rest_args)))
+ (when (not (= minarg nonrest))
+ (cl-loop for i from minarg below nonrest
+ for bb = (intern (format "entry_fallback_%s" i))
+ for next-bb = (if (= (1+ i) nonrest)
+ 'entry_rest_args
+ (intern (format "entry_fallback_%s" (1+ i))))
+ do (comp-with-sp i
+ (comp-make-curr-block bb (comp-sp))
+ (comp-emit-set-const nil)
+ (comp-emit `(jump ,next-bb)))))
+ (comp-make-curr-block 'entry_rest_args (comp-sp))
+ (comp-emit `(set-rest-args-to-local ,(comp-slot-n nonrest)))
+ (setf (comp-sp) nonrest)
+ (when (and (> nonrest 8) (null rest))
+ (cl-decf (comp-sp))))
+
+(defun comp-limplify-finalize-function (func)
+ "Reverse insns into all basic blocks of FUNC."
+ (cl-loop for bb being the hash-value in (comp-func-blocks func)
+ do (setf (comp-block-insns bb)
+ (nreverse (comp-block-insns bb))))
+ (comp-log-func func 2)
+ func)
+
+(cl-defgeneric comp-emit-for-top-level (form)
+ "Emit the limple code for top level FORM.")
+
+(cl-defmethod comp-emit-for-top-level ((form byte-to-native-function))
+ (let* ((name (byte-to-native-function-name form))
+ (f (gethash name (comp-ctxt-funcs-h comp-ctxt)))
+ (args (comp-func-args f))
+ (c-name (comp-func-c-name f))
+ (doc (comp-func-doc f)))
+ (cl-assert (and name f))
+ (comp-emit (comp-call 'comp--register-subr
+ (make-comp-mvar :constant name)
+ (make-comp-mvar :constant (comp-args-base-min args))
+ (make-comp-mvar :constant (if (comp-args-p args)
+ (comp-args-max args)
+ 'many))
+ (make-comp-mvar :constant c-name)
+ (make-comp-mvar :constant doc)))))
+
+(cl-defmethod comp-emit-for-top-level ((form byte-to-native-top-level))
+ (let ((form (byte-to-native-top-level-form form)))
+ (comp-emit (comp-call 'eval
+ (make-comp-mvar :constant form)
+ (make-comp-mvar :constant t)))))
+
+(defun comp-limplify-top-level ()
+ "Create a limple function doing the business for top level forms.
+This will be called at load-time."
+ (let* ((func (make-comp-func :name 'top-level-run
+ :c-name "top_level_run"
+ :args (make-comp-args :min 0 :max 0)
+ :frame-size 0))
+ (comp-func func)
+ (comp-pass (make-comp-limplify
+ :curr-block (make--comp-block -1 0 'top-level)
+ :frame (comp-new-frame 0))))
+ (comp-make-curr-block 'entry (comp-sp))
+ (comp-emit-annotation "Top level")
+ (mapc #'comp-emit-for-top-level (comp-ctxt-top-level-forms comp-ctxt))
+ (comp-emit `(return ,(make-comp-mvar :constant t)))
+ (comp-limplify-finalize-function func)))
+
+(defun comp-addr-to-bb-name (addr)
+ "Search for a block starting at ADDR into pending or limplified blocks."
+ ;; FIXME Actually we could have another hash for this.
+ (cl-flet ((pred (bb)
+ (equal (comp-block-addr bb) addr)))
+ (if-let ((pending (cl-find-if #'pred
+ (comp-limplify-pending-blocks comp-pass))))
+ (comp-block-name pending)
+ (cl-loop for bb being the hash-value in (comp-func-blocks comp-func)
+ when (pred bb)
+ return (comp-block-name bb)))))
+
+(defun comp-limplify-block (bb)
+ "Limplify basic-block BB and add it to the current function."
+ (setf (comp-limplify-curr-block comp-pass) bb
+ (comp-limplify-sp comp-pass) (comp-block-sp bb)
+ (comp-limplify-pc comp-pass) (comp-block-addr bb))
+ (puthash (comp-block-name bb) bb (comp-func-blocks comp-func))
+ (cl-loop
+ for inst-cell on (nthcdr (comp-limplify-pc comp-pass)
+ (comp-func-lap comp-func))
+ for inst = (car inst-cell)
+ for next-inst = (car-safe (cdr inst-cell))
+ do (comp-limplify-lap-inst inst)
+ (cl-incf (comp-limplify-pc comp-pass))
+ when (comp-lap-fall-through-p inst)
+ do (pcase next-inst
+ (`(TAG ,_label . ,label-sp)
+ (when label-sp
+ (cl-assert (= (1- label-sp) (comp-sp))))
+ (let* ((stack-depth (if label-sp
+ (1- label-sp)
+ (comp-sp)))
+ (next-bb (comp-block-name (comp-bb-maybe-add (comp-limplify-pc comp-pass) stack-depth))))
+ (unless (comp-block-closed bb)
+ (comp-emit `(jump ,next-bb))))
+ (cl-return)))
+ until (comp-lap-eob-p inst)))
+
+(defun comp-limplify-function (func)
+ "Limplify a single function FUNC."
+ (let* ((frame-size (comp-func-frame-size func))
+ (comp-func func)
+ (comp-pass (make-comp-limplify
+ :frame (comp-new-frame frame-size)))
+ (args (comp-func-args func)))
+ (comp-fill-label-h)
+ ;; Prologue
+ (comp-make-curr-block 'entry (comp-sp))
+ (comp-emit-annotation (concat "Lisp function: "
+ (symbol-name (comp-func-name func))))
+ (if (comp-args-p args)
+ (cl-loop for i below (comp-args-max args)
+ do (cl-incf (comp-sp))
+ (comp-emit `(set-par-to-local ,(comp-slot) ,i)))
+ (comp-emit-narg-prologue (comp-args-base-min args)
+ (comp-nargs-nonrest args)
+ (comp-nargs-rest args)))
+ (comp-emit '(jump bb_0))
+ ;; Body
+ (comp-bb-maybe-add 0 (comp-sp))
+ (cl-loop for next-bb = (pop (comp-limplify-pending-blocks comp-pass))
+ while next-bb
+ do (comp-limplify-block next-bb))
+ ;; Sanity check against block duplication.
+ (cl-loop with addr-h = (make-hash-table)
+ for bb being the hash-value in (comp-func-blocks func)
+ for addr = (comp-block-addr bb)
+ when addr
+ do (cl-assert (null (gethash addr addr-h)))
+ (puthash addr t addr-h))
+ (comp-limplify-finalize-function func)))
+
+(defun comp-add-func-to-ctxt (func)
+ "Add FUNC to the current compiler contex."
+ (puthash (comp-func-name func)
+ func
+ (comp-ctxt-funcs-h comp-ctxt)))
+
+(defun comp-limplify (lap-funcs)
+ "Compute the LIMPLE ir for LAP-FUNCS.
+Top level forms for the current context are rendered too."
+ (mapc #'comp-add-func-to-ctxt (mapcar #'comp-limplify-function lap-funcs))
+ (comp-add-func-to-ctxt (comp-limplify-top-level)))
+
+\f
+;;; SSA pass specific code.
+;; After limplification no edges are present between basic blocks and an
+;; implicit phi is present for every slot at the beginning of every basic block.
+;; This pass is responsible for building all the edges and replace all m-vars
+;; plus placing the needed phis.
+;; Because the number of phis placed is (supposed) to be the minimum necessary
+;; this form is called 'minimal SSA form'.
+;; This pass should be run every time basic blocks or m-var are shuffled.
+
+(cl-defun make-comp-ssa-mvar (&key slot (constant nil const-vld) type)
+ (make--comp-mvar :id (funcall (comp-func-ssa-cnt-gen comp-func))
+ :slot slot :const-vld const-vld :constant constant
+ :type type))
+
+(defun comp-compute-edges ()
+ "Compute the basic block edges for the current function."
+ (cl-flet ((edge-add (&rest args)
+ (push
+ (apply #'make--comp-edge
+ :number (funcall (comp-func-edge-cnt-gen comp-func))
+ args)
+ (comp-func-edges comp-func))))
+
+ (cl-loop with blocks = (comp-func-blocks comp-func)
+ for bb being each hash-value of blocks
+ for last-insn = (car (last (comp-block-insns bb)))
+ for (op first second third forth) = last-insn
+ do (cl-case op
+ (jump
+ (edge-add :src bb :dst (gethash first blocks)))
+ (cond-jump
+ (edge-add :src bb :dst (gethash third blocks))
+ (edge-add :src bb :dst (gethash forth blocks)))
+ (cond-jump-narg-leq
+ (edge-add :src bb :dst (gethash second blocks))
+ (edge-add :src bb :dst (gethash third blocks)))
+ (push-handler
+ (edge-add :src bb :dst (gethash third blocks))
+ (edge-add :src bb :dst (gethash forth blocks)))
+ (return)
+ (otherwise
+ (signal 'native-ice
+ (list "block does not end with a branch"
+ bb
+ (comp-func-name comp-func)))))
+ finally (setf (comp-func-edges comp-func)
+ (nreverse (comp-func-edges comp-func)))
+ ;; Update edge refs into blocks.
+ (cl-loop for edge in (comp-func-edges comp-func)
+ do (push edge
+ (comp-block-out-edges (comp-edge-src edge)))
+ (push edge
+ (comp-block-in-edges (comp-edge-dst edge))))
+ (comp-log-edges comp-func))))
+
+(defun comp-collect-rev-post-order (basic-block)
+ "Walk BASIC-BLOCK children and return their name in reversed post-order."
+ (let ((visited (make-hash-table))
+ (acc ()))
+ (cl-labels ((collect-rec (bb)
+ (let ((name (comp-block-name bb)))
+ (unless (gethash name visited)
+ (puthash name t visited)
+ (cl-loop for e in (comp-block-out-edges bb)
+ for dst-block = (comp-edge-dst e)
+ do (collect-rec dst-block))
+ (push name acc)))))
+ (collect-rec basic-block)
+ acc)))
+
+(defun comp-compute-dominator-tree ()
+ "Compute immediate dominators for each basic block in current function."
+ ;; Originally based on: "A Simple, Fast Dominance Algorithm"
+ ;; Cooper, Keith D.; Harvey, Timothy J.; Kennedy, Ken (2001).
+ (cl-flet ((intersect (b1 b2)
+ (let ((finger1 (comp-block-post-num b1))
+ (finger2 (comp-block-post-num b2)))
+ (while (not (= finger1 finger2))
+ (while (< finger1 finger2)
+ (setf b1 (comp-block-dom b1)
+ finger1 (comp-block-post-num b1)))
+ (while (< finger2 finger1)
+ (setf b2 (comp-block-dom b2)
+ finger2 (comp-block-post-num b2))))
+ b1))
+ (first-processed (l)
+ (if-let ((p (cl-find-if (lambda (p) (comp-block-dom p)) l)))
+ p
+ (signal 'native-ice "cant't find first preprocessed"))))
+
+ (when-let ((blocks (comp-func-blocks comp-func))
+ (entry (gethash 'entry blocks))
+ ;; No point to go on if the only bb is 'entry'.
+ (bb1 (gethash 'bb_1 blocks)))
+ (cl-loop with rev-bb-list = (comp-collect-rev-post-order entry)
+ with changed = t
+ while changed
+ initially (progn
+ (comp-log "Computing dominator tree...\n" 2)
+ (setf (comp-block-dom entry) entry)
+ ;; Set the post order number.
+ (cl-loop for name in (reverse rev-bb-list)
+ for b = (gethash name blocks)
+ for i from 0
+ do (setf (comp-block-post-num b) i)))
+ do (cl-loop
+ for name in (cdr rev-bb-list)
+ for b = (gethash name blocks)
+ for preds = (comp-block-preds b)
+ for new-idom = (first-processed preds)
+ initially (setf changed nil)
+ do (cl-loop for p in (delq new-idom preds)
+ when (comp-block-dom p)
+ do (setf new-idom (intersect p new-idom)))
+ unless (eq (comp-block-dom b) new-idom)
+ do (setf (comp-block-dom b) new-idom
+ changed t))))))
+
+(defun comp-compute-dominator-frontiers ()
+ "Compute the dominator frontier for each basic block in `comp-func'."
+ ;; Originally based on: "A Simple, Fast Dominance Algorithm"
+ ;; Cooper, Keith D.; Harvey, Timothy J.; Kennedy, Ken (2001).
+ (cl-loop with blocks = (comp-func-blocks comp-func)
+ for b-name being each hash-keys of blocks
+ using (hash-value b)
+ for preds = (comp-block-preds b)
+ when (>= (length preds) 2) ; All joins
+ do (cl-loop for p in preds
+ for runner = p
+ do (while (not (eq runner (comp-block-dom b)))
+ (puthash b-name b (comp-block-df runner))
+ (setf runner (comp-block-dom runner))))))
+
+(defun comp-log-block-info ()
+ "Log basic blocks info for the current function."
+ (maphash (lambda (name bb)
+ (let ((dom (comp-block-dom bb))
+ (df (comp-block-df bb)))
+ (comp-log (format "block: %s idom: %s DF %s\n"
+ name
+ (when dom (comp-block-name dom))
+ (cl-loop for b being each hash-keys of df
+ collect b))
+ 3)))
+ (comp-func-blocks comp-func)))
+
+(defun comp-place-phis ()
+ "Place phi insns into the current function."
+ ;; Originally based on: Static Single Assignment Book
+ ;; Algorithm 3.1: Standard algorithm for inserting phi-functions
+ (cl-flet ((add-phi (slot-n bb)
+ ;; Add a phi func for slot SLOT-N at the top of BB.
+ (push `(phi ,slot-n) (comp-block-insns bb)))
+ (slot-assigned-p (slot-n bb)
+ ;; Return t if a SLOT-N was assigned within BB.
+ (cl-loop for insn in (comp-block-insns bb)
+ when (and (comp-assign-op-p (car insn))
+ (= slot-n (comp-mvar-slot (cadr insn))))
+ return t)))
+
+ (cl-loop for i from 0 below (comp-func-frame-size comp-func)
+ ;; List of blocks with a definition of mvar i
+ for defs-v = (cl-loop with blocks = (comp-func-blocks comp-func)
+ for b being each hash-value of blocks
+ when (slot-assigned-p i b)
+ collect b)
+ ;; Set of basic blocks where phi is added.
+ for f = ()
+ ;; Worklist, set of basic blocks that contain definitions of v.
+ for w = defs-v
+ do
+ (while w
+ (let ((x (pop w)))
+ (cl-loop for y being each hash-value of (comp-block-df x)
+ unless (cl-find y f)
+ do (add-phi i y)
+ (push y f)
+ ;; Adding a phi implies mentioning the
+ ;; corresponding slot so in case adjust w.
+ (unless (cl-find y defs-v)
+ (push y w))))))))
+
+(defun comp-dom-tree-walker (bb pre-lambda post-lambda)
+ "Dominator tree walker function starting from basic block BB.
+PRE-LAMBDA and POST-LAMBDA are called in pre or post-order if non nil."
+ (when pre-lambda
+ (funcall pre-lambda bb))
+ (when-let ((out-edges (comp-block-out-edges bb)))
+ (cl-loop for ed in out-edges
+ for child = (comp-edge-dst ed)
+ when (eq bb (comp-block-dom child))
+ ;; Current block is the immediate dominator then recur.
+ do (comp-dom-tree-walker child pre-lambda post-lambda)))
+ (when post-lambda
+ (funcall post-lambda bb)))
+
+(cl-defstruct (comp-ssa (:copier nil))
+ "Support structure used while SSA renaming."
+ (frame (comp-new-frame (comp-func-frame-size comp-func) t) :type vector
+ :documentation "Vector of m-vars."))
+
+(defun comp-ssa-rename-insn (insn frame)
+ (dotimes (slot-n (comp-func-frame-size comp-func))
+ (cl-flet ((targetp (x)
+ ;; Ret t if x is an mvar and target the correct slot number.
+ (and (comp-mvar-p x)
+ (eql slot-n (comp-mvar-slot x))))
+ (new-lvalue ()
+ ;; If is an assignment make a new mvar and put it as l-value.
+ (let ((mvar (make-comp-ssa-mvar :slot slot-n)))
+ (setf (aref frame slot-n) mvar
+ (cadr insn) mvar))))
+ (pcase insn
+ (`(,(pred comp-assign-op-p) ,(pred targetp) . ,_)
+ (let ((mvar (aref frame slot-n)))
+ (setcdr insn (cl-nsubst-if mvar #'targetp (cdr insn))))
+ (new-lvalue))
+ (`(phi ,n)
+ (when (equal n slot-n)
+ (new-lvalue)))
+ (_
+ (let ((mvar (aref frame slot-n)))
+ (setcdr insn (cl-nsubst-if mvar #'targetp (cdr insn)))))))))
+
+(defun comp-ssa-rename ()
+ "Entry point to rename into SSA within the current function."
+ (comp-log "Renaming\n" 2)
+ (let ((frame-size (comp-func-frame-size comp-func))
+ (visited (make-hash-table)))
+ (cl-labels ((ssa-rename-rec (bb in-frame)
+ (unless (gethash bb visited)
+ (puthash bb t visited)
+ (cl-loop for insn in (comp-block-insns bb)
+ do (comp-ssa-rename-insn insn in-frame))
+ (setf (comp-block-final-frame bb)
+ (copy-sequence in-frame))
+ (when-let ((out-edges (comp-block-out-edges bb)))
+ (cl-loop for ed in out-edges
+ for child = (comp-edge-dst ed)
+ ;; Provide a copy of the same frame to all childs.
+ do (ssa-rename-rec child (copy-sequence in-frame)))))))
+
+ (ssa-rename-rec (gethash 'entry (comp-func-blocks comp-func))
+ (comp-new-frame frame-size t)))))
+
+(defun comp-finalize-phis ()
+ "Fixup r-values into phis in all basic blocks."
+ (cl-flet ((finalize-phi (args b)
+ ;; Concatenate into args all incoming m-vars for this phi.
+ (setcdr args
+ (cl-loop with slot-n = (comp-mvar-slot (car args))
+ for e in (comp-block-in-edges b)
+ for b = (comp-edge-src e)
+ for in-frame = (comp-block-final-frame b)
+ collect (aref in-frame slot-n)))))
+
+ (cl-loop for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop for (op . args) in (comp-block-insns b)
+ when (eq op 'phi)
+ do (finalize-phi args b)))))
+
+(defun comp-ssa (_)
+ "Port all functions into mininal SSA form."
+ (maphash (lambda (_ f)
+ (let ((comp-func f))
+ ;; TODO: if this is run more than once we should clean all CFG
+ ;; data including phis here.
+ (comp-func-reset-generators comp-func)
+ (comp-compute-edges)
+ (comp-compute-dominator-tree)
+ (comp-compute-dominator-frontiers)
+ (comp-log-block-info)
+ (comp-place-phis)
+ (comp-ssa-rename)
+ (comp-finalize-phis)
+ (comp-log-func comp-func 3)))
+ (comp-ctxt-funcs-h comp-ctxt)))
+
+\f
+;;; propagate pass specific code.
+;; A very basic propagation pass follows.
+;; This propagates values and types plus ref property in the control flow graph.
+;; This is also responsible for removing function calls to pure functions if
+;; possible.
+
+(defsubst comp-strict-type-of (obj)
+ "Given OBJ return its type understanding fixnums."
+ ;; Should be certainly smarter but now we take advantages just from fixnums.
+ (if (fixnump obj)
+ 'fixnum
+ (type-of obj)))
+
+(defun comp-copy-insn (insn)
+ "Deep copy INSN."
+ ;; Adapted from `copy-tree'.
+ (if (consp insn)
+ (let (result)
+ (while (consp insn)
+ (let ((newcar (car insn)))
+ (if (or (consp (car insn)) (comp-mvar-p (car insn)))
+ (setf newcar (comp-copy-insn (car insn))))
+ (push newcar result))
+ (setf insn (cdr insn)))
+ (nconc (nreverse result)
+ (if (comp-mvar-p insn) (comp-copy-insn insn) insn)))
+ (if (comp-mvar-p insn)
+ (copy-comp-mvar insn)
+ insn)))
+
+(defun comp-basic-const-propagate ()
+ "Propagate simple constants for setimm operands.
+This can run just once."
+ (cl-loop
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop
+ for insn in (comp-block-insns b)
+ do (pcase insn
+ (`(setimm ,lval ,_ ,v)
+ (setf (comp-mvar-const-vld lval) t
+ (comp-mvar-constant lval) v
+ (comp-mvar-type lval) (comp-strict-type-of v)))))))
+
+(defsubst comp-mvar-propagate (lval rval)
+ "Propagate into LVAL properties of RVAL."
+ (setf (comp-mvar-const-vld lval) (comp-mvar-const-vld rval)
+ (comp-mvar-constant lval) (comp-mvar-constant rval)
+ (comp-mvar-type lval) (comp-mvar-type rval)))
+
+(defsubst comp-function-call-remove (insn f args)
+ "Given INSN when F is pure if all ARGS are known remove the function call."
+ (when (and (get f 'pure) ; Can we just optimize pure here? See byte-opt.el
+ (cl-every #'comp-mvar-const-vld args))
+ (let ((val (apply f (mapcar #'comp-mvar-constant args))))
+ ;; See `comp-emit-set-const'.
+ (setf (car insn) 'setimm
+ (cddr insn) (list (comp-add-const-to-relocs val) val)))))
+
+(defun comp-propagate-insn (insn)
+ "Propagate within INSN."
+ (pcase insn
+ (`(set ,lval ,rval)
+ (pcase rval
+ (`(,(or 'call 'direct-call) ,f . ,args)
+ (setf (comp-mvar-type lval)
+ (alist-get f comp-known-ret-types))
+ (comp-function-call-remove insn f args))
+ (`(,(or 'callref 'direct-callref) ,f . ,args)
+ (cl-loop for v in args
+ do (setf (comp-mvar-ref v) t))
+ (setf (comp-mvar-type lval)
+ (alist-get f comp-known-ret-types))
+ (comp-function-call-remove insn f args))
+ (_
+ (comp-mvar-propagate lval rval))))
+ (`(phi ,lval . ,rest)
+ ;; Const prop here.
+ (when-let* ((vld (cl-every #'comp-mvar-const-vld rest))
+ (consts (mapcar #'comp-mvar-constant rest))
+ (x (car consts))
+ (equals (cl-every (lambda (y) (equal x y)) consts)))
+ (setf (comp-mvar-constant lval) x))
+ ;; Type propagation.
+ ;; FIXME: checking for type equality is not sufficient cause does not
+ ;; account type hierarchy!
+ (when-let* ((types (mapcar #'comp-mvar-type rest))
+ (non-empty (cl-notany #'null types))
+ (x (car types))
+ (eqs (cl-every (lambda (y) (eq x y)) types)))
+ (setf (comp-mvar-type lval) x))
+ ;; Reference propagation.
+ (let ((operands (cons lval rest)))
+ (when (cl-some #'comp-mvar-ref operands)
+ (mapc (lambda (x) (setf (comp-mvar-ref x) t)) operands))))))
+
+(defun comp-propagate* ()
+ "Propagate for set* and phi operands.
+Return t if something was changed."
+ (cl-loop with modified = nil
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop for insn in (comp-block-insns b)
+ for orig-insn = (unless modified ; Save consing after 1th change.
+ (comp-copy-insn insn))
+ do (comp-propagate-insn insn)
+ when (and (null modified) (not (equal insn orig-insn)))
+ do (setf modified t))
+ finally return modified))
+
+(defun comp-propagate (_)
+ (when (>= comp-speed 2)
+ (maphash (lambda (_ f)
+ (let ((comp-func f))
+ (comp-basic-const-propagate)
+ (cl-loop
+ for i from 1
+ while (comp-propagate*)
+ finally (comp-log (format "Propagation run %d times\n" i) 2))
+ (comp-log-func comp-func 3)))
+ (comp-ctxt-funcs-h comp-ctxt))))
+
+\f
+;;; Call optimizer pass specific code.
+;; This pass is responsible for the following optimizations:
+;; - Call to subrs that are in defined in the C source and are passing through
+;; funcall trampoline gets optimized into normal indirect calls.
+;; This makes effectively this calls equivalent to all the subrs that got
+;; dedicated byte-code ops.
+;; Triggered at comp-speed >= 2.
+;; - Recursive calls gets optimized into direct calls.
+;; Triggered at comp-speed >= 2.
+;; - Intra compilation unit procedure calls gets optimized into direct calls.
+;; This can be a big win and even allow gcc to inline but does not make
+;; function in the compilation unit re-definable safely without recompiling
+;; the full compilation unit.
+;; For this reason this is triggered only at comp-speed == 3.
+
+(defun comp-call-optim-form-call (callee args self)
+ ""
+ (cl-flet ((fill-args (args total)
+ ;; Fill missing args to reach TOTAL
+ (append args (cl-loop repeat (- total (length args))
+ collect (make-comp-mvar :constant nil))))
+ (clean-args-ref (args)
+ ;; Clean-up the ref slot in all args
+ (mapc (lambda (arg)
+ (setf (comp-mvar-ref arg) nil))
+ args)
+ args))
+ (when (symbolp callee) ; Do nothing if callee is a byte compiled func.
+ (let* ((f (symbol-function callee))
+ (subrp (subrp f))
+ (callee-in-unit (gethash callee
+ (comp-ctxt-funcs-h comp-ctxt))))
+ (cond
+ ((and subrp (not (subr-native-elisp-p f)))
+ ;; Trampoline removal.
+ (let* ((maxarg (cdr (subr-arity f)))
+ (call-type (if (if subrp
+ (not (numberp maxarg))
+ (comp-nargs-p callee-in-unit))
+ 'callref
+ 'call))
+ (args (if (eq call-type 'callref)
+ args
+ (fill-args args maxarg))))
+ (comp-add-subr-to-relocs callee)
+ `(,call-type ,callee ,@(clean-args-ref args))))
+ ;; Intra compilation unit procedure call optimization.
+ ;; Attention speed 3 triggers that for non self calls too!!
+ ((or (eq callee self)
+ (and (>= comp-speed 3)
+ callee-in-unit))
+ (let* ((func-args (comp-func-args callee-in-unit))
+ (nargs (comp-nargs-p func-args))
+ (call-type (if nargs 'direct-callref 'direct-call))
+ (args (if (eq call-type 'direct-callref)
+ args
+ (fill-args args (comp-args-max func-args)))))
+ `(,call-type ,callee ,@(clean-args-ref args))))
+ ((comp-type-hint-p callee)
+ `(call ,callee ,@args)))))))
+
+(defun comp-call-optim-func ()
+ "Perform the trampoline call optimization for the current function."
+ (cl-loop
+ with self = (comp-func-name comp-func)
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop
+ for insn-cell on (comp-block-insns b)
+ for insn = (car insn-cell)
+ do (pcase insn
+ (`(set ,lval (callref funcall ,f . ,rest))
+ (when-let ((new-form (comp-call-optim-form-call
+ (comp-mvar-constant f) rest self)))
+ (setcar insn-cell `(set ,lval ,new-form))))
+ (`(callref funcall ,f . ,rest)
+ (when-let ((new-form (comp-call-optim-form-call
+ (comp-mvar-constant f) rest self)))
+ (setcar insn-cell new-form)))))))
+
+(defun comp-call-optim (_)
+ "Try to optimize out funcall trampoline usage when possible."
+ (when (>= comp-speed 2)
+ (maphash (lambda (_ f)
+ (let ((comp-func f))
+ (comp-call-optim-func)))
+ (comp-ctxt-funcs-h comp-ctxt))))
+
+\f
+;;; Dead code elimination pass specific code.
+;; This simple pass try to eliminate insns became useful after propagation.
+;; Even if gcc would take care of this is good to perform this here
+;; in the hope of removing memory references.
+;;
+;; This pass can be run as last optim.
+
+(defun comp-collect-mvar-ids (insn)
+ "Collect the m-var unique identifiers into INSN."
+ (cl-loop for x in insn
+ if (consp x)
+ append (comp-collect-mvar-ids x)
+ else
+ when (comp-mvar-p x)
+ collect (comp-mvar-id x)))
+
+(defun comp-dead-assignments-func ()
+ "Clean-up dead assignments into current function.
+Return the list of m-var ids nuked."
+ (let ((l-vals ())
+ (r-vals ()))
+ ;; Collect used r and l-values.
+ (cl-loop
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop
+ for insn in (comp-block-insns b)
+ for (op arg0 . rest) = insn
+ if (comp-set-op-p op)
+ do (push (comp-mvar-id arg0) l-vals)
+ (setf r-vals (nconc (comp-collect-mvar-ids rest) r-vals))
+ else
+ do (setf r-vals (nconc (comp-collect-mvar-ids insn) r-vals))))
+ ;; Every l-value appearing that does not appear as r-value has no right to
+ ;; exist and gets nuked.
+ (let ((nuke-list (cl-set-difference l-vals r-vals)))
+ (comp-log (format "Function %s\nl-vals %s\nr-vals %s\nNuking ids: %s\n"
+ (comp-func-name comp-func)
+ l-vals
+ r-vals
+ nuke-list)
+ 3)
+ (cl-loop
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop
+ for insn-cell on (comp-block-insns b)
+ for insn = (car insn-cell)
+ for (op arg0 rest) = insn
+ when (and (comp-set-op-p op)
+ (member (comp-mvar-id arg0) nuke-list))
+ do (setcar insn-cell
+ (if (comp-limple-insn-call-p rest)
+ rest
+ `(comment ,(format "optimized out: %s"
+ insn))))))
+ nuke-list)))
+
+(defun comp-remove-type-hints-func ()
+ "Remove type hints from the current function.
+These are substituted with a normal 'set' op."
+ (cl-loop
+ for b being each hash-value of (comp-func-blocks comp-func)
+ do (cl-loop
+ for insn-cell on (comp-block-insns b)
+ for insn = (car insn-cell)
+ do (pcase insn
+ (`(set ,l-val (call ,(pred comp-type-hint-p) ,r-val))
+ (setcar insn-cell `(set ,l-val ,r-val)))))))
+
+(defun comp-dead-code (_)
+ "Dead code elimination."
+ (when (>= comp-speed 2)
+ (maphash (lambda (_ f)
+ (let ((comp-func f))
+ (cl-loop
+ for i from 1
+ while (comp-dead-assignments-func)
+ finally (comp-log (format "dead code rm run %d times\n" i) 2)
+ (comp-log-func comp-func 3))
+ (comp-remove-type-hints-func)
+ (comp-log-func comp-func 3)))
+ (comp-ctxt-funcs-h comp-ctxt))))
+
+\f
+;;; Final pass specific code.
+
+(defun comp-compile-ctxt-to-file (name)
+ "Compile as native code the current context naming it NAME.
+Prepare every function for final compilation and drive the C back-end."
+ (cl-assert (= (length (comp-ctxt-data-relocs-l comp-ctxt))
+ (hash-table-count (comp-ctxt-data-relocs-idx comp-ctxt))))
+ (comp--compile-ctxt-to-file name))
+
+(defun comp-final (_)
+ "Final pass driving the C back-end for code emission."
+ (let (compile-result)
+ (maphash (lambda (_ f)
+ (comp-log-func f 1))
+ (comp-ctxt-funcs-h comp-ctxt))
+ (comp--init-ctxt)
+ (unwind-protect
+ (setf compile-result
+ (comp-compile-ctxt-to-file (comp-ctxt-output comp-ctxt)))
+ (and (comp--release-ctxt)
+ compile-result))))
+
+\f
+;;; Compiler type hints.
+;; These are public entry points be used in user code to give comp suggestion
+;; about types.
+;; These can be used to implement CL style 'the', 'declare' or something like.
+;; Note: types will propagates.
+;; WARNING: At speed >= 2 type checking is not performed anymore and suggestions
+;; are assumed just to be true. Use with extreme caution...
+
+(defun comp-hint-fixnum (x)
+ (unless (fixnump x)
+ (signal 'wrong-type-argument x)))
+
+(defun comp-hint-cons (x)
+ (unless (consp x)
+ (signal 'wrong-type-argument x)))
+
+\f
+;; Some entry point support code.
+
+(defvar comp-src-pool ()
+ "List containing the files to be compiled.")
+
+(defvar comp-prc-pool ()
+ "List containing all async compilation processes.")
+
+(defun comp-to-file-p (file)
+ "Return t if FILE has to be compiled."
+ (let ((compiled-f (concat file "n")))
+ (or comp-always-compile
+ (not (and (file-exists-p compiled-f)
+ (file-newer-than-file-p compiled-f file))))))
+
+(cl-defun comp-start-async-worker ()
+ "Run an async compile worker."
+ (let (f)
+ (while (setf f (pop comp-src-pool))
+ (when (comp-to-file-p f)
+ (let* ((code `(progn
+ (require 'comp)
+ (setf comp-speed ,comp-speed
+ comp-debug ,comp-debug
+ comp-verbose ,comp-verbose
+ load-path ',load-path)
+ (message "Compiling %s started." ,f)
+ (native-compile ,f))))
+ (push (make-process :name (concat "Compiling: " f)
+ :buffer (get-buffer-create comp-async-buffer-name)
+ :command (list (concat invocation-directory
+ invocation-name)
+ "--batch"
+ "--eval"
+ (prin1-to-string code))
+ :sentinel (lambda (prc _event)
+ (accept-process-output prc)
+ (comp-start-async-worker)))
+ comp-prc-pool)
+ (cl-return-from comp-start-async-worker))))
+ (when (cl-notany #'process-live-p comp-prc-pool)
+ (let ((msg "Compilation finished."))
+ (setf comp-prc-pool ())
+ (with-current-buffer (get-buffer-create comp-async-buffer-name)
+ (save-excursion
+ (goto-char (point-max))
+ (insert msg "\n")))
+ (message msg)))))
+\f
+;;; Compiler entry points.
+
+;;;###autoload
+(defun native-compile (input)
+ "Compile INPUT into native code.
+This is the entry-point for the Emacs Lisp native compiler.
+If INPUT is a symbol, native compile its function definition.
+If INPUT is a string, use it as the file path to be native compiled.
+Return the compilation unit file name."
+ (unless (or (symbolp input)
+ (stringp input))
+ (signal 'native-compiler-error
+ (list "not a symbol function or file" input)))
+ (let ((data input)
+ (comp-native-compiling t)
+ (comp-ctxt (make-comp-ctxt
+ :output (if (symbolp input)
+ (make-temp-file (concat (symbol-name input) "-"))
+ (file-name-sans-extension (expand-file-name input))))))
+ (comp-log "\n\f\n" 1)
+ (condition-case err
+ (mapc (lambda (pass)
+ (comp-log (format "Running pass %s:\n" pass) 2)
+ (setf data (funcall pass data)))
+ comp-passes)
+ (native-compiler-error
+ ;; Add source input.
+ (let ((err-val (cdr err)))
+ (signal (car err) (if (consp err-val)
+ (cons input err-val)
+ (list input err-val))))))
+ data))
+
+;;;###autoload
+(defun native-compile-async (input &optional jobs recursively)
+ "Compile INPUT asynchronously.
+INPUT can be either a list of files a folder or a file.
+JOBS specifies the number of jobs (commands) to run simultaneously (1 default).
+Follow folders RECURSIVELY if non nil."
+ (let ((jobs (or jobs 1))
+ (files (if (listp input)
+ input
+ (if (file-directory-p input)
+ (if recursively
+ (directory-files-recursively input "\\.el$")
+ (directory-files input t "\\.el$"))
+ (if (file-exists-p input)
+ (list input)
+ (signal 'native-compiler-error
+ "input not a file nor directory"))))))
+ (setf comp-src-pool (nconc files comp-src-pool))
+ (cl-loop repeat jobs
+ do (comp-start-async-worker))
+ (message "Compilation started.")))
+
+(provide 'comp)
+
+;;; comp.el ends here
diff --git a/src/Makefile.in b/src/Makefile.in
index 76aa6a1..fd189e9 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -326,6 +326,10 @@ GETLOADAVG_LIBS =
GMP_LIB = @GMP_LIB@
GMP_OBJ = @GMP_OBJ@
+LIBGCCJIT = @LIBGCCJIT_LIB@
+## dynlib.o comp.o if native compiler is enabled, else empty
+COMP_OBJ = @COMP_OBJ@
+
RUN_TEMACS = ./temacs
# Whether builds should contain details. '--no-build-details' or empty.
@@ -414,7 +418,7 @@ base_obj =
cmds.o casetab.o casefiddle.o indent.o search.o regex-emacs.o undo.o \
alloc.o pdumper.o data.o doc.o editfns.o callint.o \
eval.o floatfns.o fns.o font.o print.o lread.o $(MODULES_OBJ) \
- syntax.o $(UNEXEC_OBJ) bytecode.o \
+ syntax.o $(UNEXEC_OBJ) bytecode.o $(COMP_OBJ) \
process.o gnutls.o callproc.o \
region-cache.o sound.o timefns.o atimer.o \
doprnt.o intervals.o textprop.o composite.o xml.o lcms.o $(NOTIFY_OBJ) \
@@ -531,7 +535,7 @@ LIBES =
$(FREETYPE_LIBS) $(FONTCONFIG_LIBS) $(HARFBUZZ_LIBS) $(LIBOTF_LIBS) $(M17N_FLT_LIBS) \
$(LIBGNUTLS_LIBS) $(LIB_PTHREAD) $(GETADDRINFO_A_LIBS) $(LCMS2_LIBS) \
$(NOTIFY_LIBS) $(LIB_MATH) $(LIBZ) $(LIBMODULES) $(LIBSYSTEMD_LIBS) \
- $(JSON_LIBS) $(GMP_LIB)
+ $(JSON_LIBS) $(GMP_LIB) $(LIBGCCJIT)
## FORCE it so that admin/unidata can decide whether this file is
## up-to-date. Although since charprop depends on bootstrap-emacs,
diff --git a/src/comp.c b/src/comp.c
new file mode 100644
index 0000000..e2629de
--- /dev/null
+++ b/src/comp.c
@@ -0,0 +1,3480 @@
+/* Compile elisp into native code.
+ Copyright (C) 2019 Free Software Foundation, Inc.
+
+Author: Andrea Corallo <akrl@sdf.org>
+
+This file is part of GNU Emacs.
+
+GNU Emacs is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or (at
+your option) any later version.
+
+GNU Emacs is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. */
+
+#include <config.h>
+
+#ifdef HAVE_NATIVE_COMP
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <libgccjit.h>
+
+#include "lisp.h"
+#include "puresize.h"
+#include "window.h"
+#include "dynlib.h"
+#include "buffer.h"
+#include "blockinput.h"
+
+/* C symbols emitted for the load relocation mechanism. */
+#define CURRENT_THREAD_RELOC_SYM "current_thread_reloc"
+#define PURE_RELOC_SYM "pure_reloc"
+#define DATA_RELOC_SYM "d_reloc"
+#define IMPORTED_FUNC_RELOC_SYM "f_reloc"
+#define TEXT_DATA_RELOC_SYM "text_data_reloc"
+#define TEXT_IMPORTED_FUNC_RELOC_SYM "text_imported_funcs"
+
+#define SPEED XFIXNUM (Fsymbol_value (Qcomp_speed))
+#define COMP_DEBUG XFIXNUM (Fsymbol_value (Qcomp_debug))
+
+#define STR_VALUE(s) #s
+#define STR(s) STR_VALUE (s)
+
+#define FIRST(x) \
+ XCAR(x)
+#define SECOND(x) \
+ XCAR (XCDR (x))
+#define THIRD(x) \
+ XCAR (XCDR (XCDR (x)))
+
+/* Like call1 but stringify and intern. */
+#define CALL1I(fun, arg) \
+ CALLN (Ffuncall, intern_c_string (STR (fun)), arg)
+
+#define DECL_BLOCK(name, func) \
+ gcc_jit_block *(name) = \
+ gcc_jit_function_new_block ((func), STR (name))
+
+#ifdef HAVE__SETJMP
+#define SETJMP _setjmp
+#else
+#define SETJMP setjmp
+#endif
+#define SETJMP_NAME STR (SETJMP)
+
+/* C side of the compiler context. */
+
+typedef struct {
+ gcc_jit_context *ctxt;
+ gcc_jit_type *void_type;
+ gcc_jit_type *bool_type;
+ gcc_jit_type *char_type;
+ gcc_jit_type *int_type;
+ gcc_jit_type *unsigned_type;
+ gcc_jit_type *long_type;
+ gcc_jit_type *unsigned_long_type;
+ gcc_jit_type *long_long_type;
+ gcc_jit_type *unsigned_long_long_type;
+ gcc_jit_type *emacs_int_type;
+ gcc_jit_type *void_ptr_type;
+ gcc_jit_type *char_ptr_type;
+ gcc_jit_type *ptrdiff_type;
+ gcc_jit_type *uintptr_type;
+ gcc_jit_type *lisp_obj_type;
+ gcc_jit_type *lisp_obj_ptr_type;
+ gcc_jit_field *lisp_obj_as_ptr;
+ gcc_jit_field *lisp_obj_as_num;
+ /* struct Lisp_Cons */
+ gcc_jit_struct *lisp_cons_s;
+ gcc_jit_field *lisp_cons_u;
+ gcc_jit_field *lisp_cons_u_s;
+ gcc_jit_field *lisp_cons_u_s_car;
+ gcc_jit_field *lisp_cons_u_s_u;
+ gcc_jit_field *lisp_cons_u_s_u_cdr;
+ gcc_jit_type *lisp_cons_type;
+ gcc_jit_type *lisp_cons_ptr_type;
+ /* struct jmp_buf. */
+ gcc_jit_struct *jmp_buf_s;
+ /* struct handler. */
+ gcc_jit_struct *handler_s;
+ gcc_jit_field *handler_jmp_field;
+ gcc_jit_field *handler_val_field;
+ gcc_jit_field *handler_next_field;
+ gcc_jit_type *handler_ptr_type;
+ gcc_jit_lvalue *loc_handler;
+ /* struct thread_state. */
+ gcc_jit_struct *thread_state_s;
+ gcc_jit_field *m_handlerlist;
+ gcc_jit_type *thread_state_ptr_type;
+ gcc_jit_rvalue *current_thread_ref;
+ /* Other globals. */
+ gcc_jit_rvalue *pure_ref;
+ /* libgccjit has really limited support for casting therefore this union will
+ be used for the scope. */
+ gcc_jit_type *cast_union_type;
+ gcc_jit_field *cast_union_as_ll;
+ gcc_jit_field *cast_union_as_ull;
+ gcc_jit_field *cast_union_as_l;
+ gcc_jit_field *cast_union_as_ul;
+ gcc_jit_field *cast_union_as_u;
+ gcc_jit_field *cast_union_as_i;
+ gcc_jit_field *cast_union_as_b;
+ gcc_jit_field *cast_union_as_uintptr;
+ gcc_jit_field *cast_union_as_ptrdiff;
+ gcc_jit_field *cast_union_as_c_p;
+ gcc_jit_field *cast_union_as_v_p;
+ gcc_jit_field *cast_union_as_lisp_cons_ptr;
+ gcc_jit_field *cast_union_as_lisp_obj;
+ gcc_jit_field *cast_union_as_lisp_obj_ptr;
+ gcc_jit_function *func; /* Current function being compiled. */
+ gcc_jit_block *block; /* Current basic block being compiled. */
+ gcc_jit_lvalue **frame; /* Frame for the current function. */
+ gcc_jit_lvalue **f_frame; /* "Floating" frame for the current function. */
+ gcc_jit_lvalue *scratch; /* Used as scratch slot for some code sequence (switch). */
+ gcc_jit_rvalue *most_positive_fixnum;
+ gcc_jit_rvalue *most_negative_fixnum;
+ gcc_jit_rvalue *one;
+ gcc_jit_rvalue *inttypebits;
+ gcc_jit_rvalue *lisp_int0;
+ gcc_jit_function *pseudovectorp;
+ gcc_jit_function *bool_to_lisp_obj;
+ gcc_jit_function *add1;
+ gcc_jit_function *sub1;
+ gcc_jit_function *negate;
+ gcc_jit_function *car;
+ gcc_jit_function *cdr;
+ gcc_jit_function *setcar;
+ gcc_jit_function *setcdr;
+ gcc_jit_function *check_type;
+ gcc_jit_function *check_impure;
+ Lisp_Object func_blocks_h; /* blk_name -> gcc_block. */
+ Lisp_Object exported_funcs_h; /* subr_name -> gcc_jit_function *. */
+ Lisp_Object imported_funcs_h; /* subr_name -> reloc_field. */
+ Lisp_Object emitter_dispatcher;
+ gcc_jit_rvalue *data_relocs; /* Synthesized struct holding data relocs. */
+ gcc_jit_lvalue *func_relocs; /* Synthesized struct holding func relocs. */
+} comp_t;
+
+static comp_t comp;
+
+FILE *logfile = NULL;
+
+/* This is used for serialized objects by the reload mechanism. */
+typedef struct {
+ ptrdiff_t len;
+ const char data[];
+} static_obj_t;
+
+\f
+/*
+ Helper functions called by the run-time.
+*/
+Lisp_Object helper_save_window_excursion (Lisp_Object v1);
+void helper_unwind_protect (Lisp_Object handler);
+Lisp_Object helper_temp_output_buffer_setup (Lisp_Object x);
+Lisp_Object helper_unbind_n (Lisp_Object n);
+void helper_save_restriction (void);
+bool helper_PSEUDOVECTOR_TYPEP_XUNTAG (Lisp_Object a, enum pvec_type code);
+
+\f
+static char * ATTRIBUTE_FORMAT_PRINTF (1, 2)
+format_string (const char *format, ...)
+{
+ static char scratch_area[512];
+ va_list va;
+ va_start (va, format);
+ int res = vsnprintf (scratch_area, sizeof (scratch_area), format, va);
+ if (res >= sizeof (scratch_area))
+ {
+ scratch_area[sizeof (scratch_area) - 4] = '.';
+ scratch_area[sizeof (scratch_area) - 3] = '.';
+ scratch_area[sizeof (scratch_area) - 2] = '.';
+ }
+ va_end (va);
+ return scratch_area;
+}
+
+static void
+bcall0 (Lisp_Object f)
+{
+ Ffuncall (1, &f);
+}
+
+/* Try to return the original subr from `symbol' even if this was advised. */
+static Lisp_Object
+symbol_subr (Lisp_Object symbol)
+{
+ Lisp_Object maybe_subr = Fsymbol_function (symbol);
+
+ if (SUBRP (maybe_subr))
+ return maybe_subr;
+
+ if (!NILP (CALL1I (advice--p, maybe_subr)))
+ maybe_subr = CALL1I (ad-get-orig-definition, symbol);
+
+ return SUBRP (maybe_subr) ? maybe_subr : Qnil;
+}
+
+static gcc_jit_field *
+type_to_cast_field (gcc_jit_type *type)
+{
+ gcc_jit_field *field;
+
+ if (type == comp.long_long_type)
+ field = comp.cast_union_as_ll;
+ else if (type == comp.unsigned_long_long_type)
+ field = comp.cast_union_as_ull;
+ else if (type == comp.long_type)
+ field = comp.cast_union_as_l;
+ else if (type == comp.unsigned_long_type)
+ field = comp.cast_union_as_ul;
+ else if (type == comp.unsigned_type)
+ field = comp.cast_union_as_u;
+ else if (type == comp.int_type)
+ field = comp.cast_union_as_i;
+ else if (type == comp.bool_type)
+ field = comp.cast_union_as_b;
+ else if (type == comp.void_ptr_type)
+ field = comp.cast_union_as_v_p;
+ else if (type == comp.uintptr_type)
+ field = comp.cast_union_as_uintptr;
+ else if (type == comp.ptrdiff_type)
+ field = comp.cast_union_as_ptrdiff;
+ else if (type == comp.char_ptr_type)
+ field = comp.cast_union_as_c_p;
+ else if (type == comp.lisp_cons_ptr_type)
+ field = comp.cast_union_as_lisp_cons_ptr;
+ else if (type == comp.lisp_obj_type)
+ field = comp.cast_union_as_lisp_obj;
+ else if (type == comp.lisp_obj_ptr_type)
+ field = comp.cast_union_as_lisp_obj_ptr;
+ else
+ xsignal1 (Qnative_ice, build_string ("unsupported cast"));
+
+ return field;
+}
+
+static gcc_jit_block *
+retrive_block (Lisp_Object block_name)
+{
+ Lisp_Object value = Fgethash (block_name, comp.func_blocks_h, Qnil);
+
+ if (NILP (value))
+ xsignal1 (Qnative_ice, build_string ("missing basic block"));
+
+ return (gcc_jit_block *) xmint_pointer (value);
+}
+
+static void
+declare_block (Lisp_Object block_name)
+{
+ char *name_str = SSDATA (SYMBOL_NAME (block_name));
+ gcc_jit_block *block = gcc_jit_function_new_block (comp.func, name_str);
+ Lisp_Object value = make_mint_ptr (block);
+
+ if (!NILP (Fgethash (block_name, comp.func_blocks_h, Qnil)))
+ xsignal1 (Qnative_ice, build_string ("double basic block declaration"));
+
+ Fputhash (block_name, value, comp.func_blocks_h);
+}
+
+static gcc_jit_lvalue *
+get_slot (Lisp_Object mvar)
+{
+ EMACS_INT slot_n = XFIXNUM (CALL1I (comp-mvar-slot, mvar));
+ if (slot_n == -1)
+ {
+ if (!comp.scratch)
+ comp.scratch = gcc_jit_function_new_local (comp.func,
+ NULL,
+ comp.lisp_obj_type,
+ "scratch");
+ return comp.scratch;
+ }
+ gcc_jit_lvalue **frame =
+ (CALL1I (comp-mvar-ref, mvar) || SPEED < 2)
+ ? comp.frame : comp.f_frame;
+ return frame[slot_n];
+}
+
+static void
+register_emitter (Lisp_Object key, void *func)
+{
+ Lisp_Object value = make_mint_ptr (func);
+ Fputhash (key, value, comp.emitter_dispatcher);
+}
+
+static void
+emit_comment (const char *str)
+{
+ if (COMP_DEBUG)
+ gcc_jit_block_add_comment (comp.block,
+ NULL,
+ str);
+}
+
+/*
+ Declare an imported function.
+ When nargs is MANY (ptrdiff_t nargs, Lisp_Object *args) signature is assumed.
+ When types is NULL args are assumed to be all Lisp_Objects.
+*/
+static gcc_jit_field *
+declare_imported_func (Lisp_Object subr_sym, gcc_jit_type *ret_type,
+ int nargs, gcc_jit_type **types)
+{
+ /* Don't want to declare the same function two times. */
+ if (!NILP (Fgethash (subr_sym, comp.imported_funcs_h, Qnil)))
+ xsignal2 (Qnative_ice,
+ build_string ("unexpected double function declaration"),
+ subr_sym);
+
+ if (nargs == MANY)
+ {
+ nargs = 2;
+ types = alloca (nargs * sizeof (* types));
+ types[0] = comp.ptrdiff_type;
+ types[1] = comp.lisp_obj_ptr_type;
+ }
+ else if (nargs == UNEVALLED)
+ {
+ nargs = 1;
+ types = alloca (nargs * sizeof (* types));
+ types[0] = comp.lisp_obj_type;
+ }
+ else if (!types)
+ {
+ types = alloca (nargs * sizeof (* types));
+ for (ptrdiff_t i = 0; i < nargs; i++)
+ types[i] = comp.lisp_obj_type;
+ }
+
+ /* String containing the function ptr name. */
+ Lisp_Object f_ptr_name =
+ CALLN (Ffuncall, intern_c_string (STR (comp-c-func-name)),
+ subr_sym, make_string ("R", 1));
+
+ gcc_jit_type *f_ptr_type =
+ gcc_jit_context_new_function_ptr_type (comp.ctxt,
+ NULL,
+ ret_type,
+ nargs,
+ types,
+ 0);
+ gcc_jit_field *field =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ f_ptr_type,
+ SSDATA (f_ptr_name));
+
+ Fputhash (subr_sym, make_mint_ptr (field), comp.imported_funcs_h);
+ return field;
+}
+
+/* Emit calls fetching from existing declarations. */
+static gcc_jit_rvalue *
+emit_call (Lisp_Object subr_sym, gcc_jit_type *ret_type, ptrdiff_t nargs,
+ gcc_jit_rvalue **args, bool direct)
+{
+ Lisp_Object func =
+ Fgethash (subr_sym, direct ? comp.exported_funcs_h: comp.imported_funcs_h,
+ Qnil);
+ if (NILP (func))
+ xsignal2 (Qnative_ice,
+ build_string ("missing function declaration"),
+ subr_sym);
+
+ if (direct)
+ {
+ emit_comment (format_string ("direct call to subr: %s",
+ SSDATA (SYMBOL_NAME (subr_sym))));
+ return gcc_jit_context_new_call (comp.ctxt,
+ NULL,
+ xmint_pointer (func),
+ nargs,
+ args);
+ }
+ else
+ {
+ gcc_jit_lvalue *f_ptr =
+ gcc_jit_lvalue_access_field (comp.func_relocs,
+ NULL,
+ (gcc_jit_field *) xmint_pointer (func));
+ if (!f_ptr)
+ xsignal2 (Qnative_ice,
+ build_string ("missing function relocation"),
+ subr_sym);
+ emit_comment (format_string ("calling subr: %s",
+ SSDATA (SYMBOL_NAME (subr_sym))));
+ return gcc_jit_context_new_call_through_ptr (comp.ctxt,
+ NULL,
+ gcc_jit_lvalue_as_rvalue (f_ptr),
+ nargs,
+ args);
+ }
+}
+
+static gcc_jit_rvalue *
+emit_call_ref (Lisp_Object subr_sym, ptrdiff_t nargs,
+ gcc_jit_lvalue *base_arg, bool direct)
+{
+ gcc_jit_rvalue *args[] =
+ { gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ nargs),
+ gcc_jit_lvalue_get_address (base_arg, NULL) };
+ return emit_call (subr_sym, comp.lisp_obj_type, 2, args, direct);
+}
+
+/* Close current basic block emitting a conditional. */
+
+static void
+emit_cond_jump (gcc_jit_rvalue *test,
+ gcc_jit_block *then_target, gcc_jit_block *else_target)
+{
+ if (gcc_jit_rvalue_get_type (test) == comp.bool_type)
+ gcc_jit_block_end_with_conditional (comp.block,
+ NULL,
+ test,
+ then_target,
+ else_target);
+ else
+ /* In case test is not bool we do a logical negation to obtain a bool as
+ result. */
+ gcc_jit_block_end_with_conditional (
+ comp.block,
+ NULL,
+ gcc_jit_context_new_unary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_UNARY_OP_LOGICAL_NEGATE,
+ comp.bool_type,
+ test),
+ else_target,
+ then_target);
+
+}
+
+static gcc_jit_rvalue *
+emit_cast (gcc_jit_type *new_type, gcc_jit_rvalue *obj)
+{
+ static ptrdiff_t i;
+
+ gcc_jit_field *orig_field =
+ type_to_cast_field (gcc_jit_rvalue_get_type (obj));
+ gcc_jit_field *dest_field = type_to_cast_field (new_type);
+
+ gcc_jit_lvalue *tmp_u =
+ gcc_jit_function_new_local (comp.func,
+ NULL,
+ comp.cast_union_type,
+ format_string ("union_cast_%td", i++));
+ gcc_jit_block_add_assignment (comp.block,
+ NULL,
+ gcc_jit_lvalue_access_field (tmp_u,
+ NULL,
+ orig_field),
+ obj);
+
+ return gcc_jit_rvalue_access_field ( gcc_jit_lvalue_as_rvalue (tmp_u),
+ NULL,
+ dest_field);
+}
+
+/*
+ Emit the equivalent of:
+ (typeof_ptr) ((uintptr) ptr + size_of_ptr_ref * i)
+*/
+
+static gcc_jit_rvalue *
+emit_ptr_arithmetic (gcc_jit_rvalue *ptr, gcc_jit_type *ptr_type,
+ int size_of_ptr_ref, gcc_jit_rvalue *i)
+{
+ emit_comment ("ptr_arithmetic");
+
+ gcc_jit_rvalue *offset =
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MULT,
+ comp.uintptr_type,
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.uintptr_type,
+ size_of_ptr_ref),
+ emit_cast (comp.uintptr_type, i));
+
+ return
+ emit_cast (
+ ptr_type,
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_PLUS,
+ comp.uintptr_type,
+ emit_cast (comp.uintptr_type, ptr),
+ offset));
+}
+
+static gcc_jit_rvalue *
+emit_XLI (gcc_jit_rvalue *obj)
+{
+ emit_comment ("XLI");
+
+ return gcc_jit_rvalue_access_field (obj,
+ NULL,
+ comp.lisp_obj_as_num);
+}
+
+static gcc_jit_lvalue *
+emit_lval_XLI (gcc_jit_lvalue *obj)
+{
+ emit_comment ("lval_XLI");
+
+ return gcc_jit_lvalue_access_field (obj,
+ NULL,
+ comp.lisp_obj_as_num);
+}
+
+/*
+static gcc_jit_rvalue *
+emit_XLP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("XLP");
+
+ return gcc_jit_rvalue_access_field (obj,
+ NULL,
+ comp.lisp_obj_as_ptr);
+}
+
+static gcc_jit_lvalue *
+emit_lval_XLP (gcc_jit_lvalue *obj)
+{
+ emit_comment ("lval_XLP");
+
+ return gcc_jit_lvalue_access_field (obj,
+ NULL,
+ comp.lisp_obj_as_ptr);
+} */
+static gcc_jit_rvalue *
+emit_XUNTAG (gcc_jit_rvalue *a, gcc_jit_type *type, ptrdiff_t lisp_word_tag)
+{
+ /* #define XUNTAG(a, type, ctype) ((ctype *)
+ ((char *) XLP (a) - LISP_WORD_TAG (type))) */
+ emit_comment ("XUNTAG");
+
+ return emit_cast (gcc_jit_type_get_pointer (type),
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MINUS,
+ comp.emacs_int_type,
+ emit_XLI (a),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ lisp_word_tag)));
+}
+
+static gcc_jit_rvalue *
+emit_XCONS (gcc_jit_rvalue *a)
+{
+ emit_comment ("XCONS");
+
+ return emit_XUNTAG (a,
+ gcc_jit_struct_as_type (comp.lisp_cons_s),
+ LISP_WORD_TAG (Lisp_Cons));
+}
+
+static gcc_jit_rvalue *
+emit_EQ (gcc_jit_rvalue *x, gcc_jit_rvalue *y)
+{
+ emit_comment ("EQ");
+
+ return gcc_jit_context_new_comparison (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_COMPARISON_EQ,
+ emit_XLI (x),
+ emit_XLI (y));
+}
+
+static gcc_jit_rvalue *
+emit_TAGGEDP (gcc_jit_rvalue *obj, ptrdiff_t tag)
+{
+ /* (! (((unsigned) (XLI (a) >> (USE_LSB_TAG ? 0 : VALBITS)) \
+ - (unsigned) (tag)) \
+ & ((1 << GCTYPEBITS) - 1))) */
+ emit_comment ("TAGGEDP");
+
+ gcc_jit_rvalue *sh_res =
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_RSHIFT,
+ comp.emacs_int_type,
+ emit_XLI (obj),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ (USE_LSB_TAG ? 0 : VALBITS)));
+
+ gcc_jit_rvalue *minus_res =
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MINUS,
+ comp.unsigned_type,
+ emit_cast (comp.unsigned_type, sh_res),
+ gcc_jit_context_new_rvalue_from_int (
+ comp.ctxt,
+ comp.unsigned_type,
+ tag));
+
+ gcc_jit_rvalue *res =
+ gcc_jit_context_new_unary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_UNARY_OP_LOGICAL_NEGATE,
+ comp.int_type,
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_BITWISE_AND,
+ comp.unsigned_type,
+ minus_res,
+ gcc_jit_context_new_rvalue_from_int (
+ comp.ctxt,
+ comp.unsigned_type,
+ ((1 << GCTYPEBITS) - 1))));
+
+ return res;
+}
+
+static gcc_jit_rvalue *
+emit_VECTORLIKEP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("VECTORLIKEP");
+
+ return emit_TAGGEDP (obj, Lisp_Vectorlike);
+}
+
+static gcc_jit_rvalue *
+emit_CONSP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("CONSP");
+
+ return emit_TAGGEDP (obj, Lisp_Cons);
+}
+
+static gcc_jit_rvalue *
+emit_FLOATP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("FLOATP");
+
+ return emit_TAGGEDP (obj, Lisp_Float);
+}
+
+static gcc_jit_rvalue *
+emit_BIGNUMP (gcc_jit_rvalue *obj)
+{
+ /* PSEUDOVECTORP (x, PVEC_BIGNUM); */
+ emit_comment ("BIGNUMP");
+
+ gcc_jit_rvalue *args[] =
+ { obj,
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.int_type,
+ PVEC_BIGNUM) };
+
+ return gcc_jit_context_new_call (comp.ctxt,
+ NULL,
+ comp.pseudovectorp,
+ 2,
+ args);
+}
+
+static gcc_jit_rvalue *
+emit_FIXNUMP (gcc_jit_rvalue *obj)
+{
+ /* (! (((unsigned) (XLI (x) >> (USE_LSB_TAG ? 0 : FIXNUM_BITS))
+ - (unsigned) (Lisp_Int0 >> !USE_LSB_TAG))
+ & ((1 << INTTYPEBITS) - 1))) */
+ emit_comment ("FIXNUMP");
+
+ gcc_jit_rvalue *sh_res =
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_RSHIFT,
+ comp.emacs_int_type,
+ emit_XLI (obj),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ (USE_LSB_TAG ? 0 : FIXNUM_BITS)));
+
+ gcc_jit_rvalue *minus_res =
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MINUS,
+ comp.unsigned_type,
+ emit_cast (comp.unsigned_type, sh_res),
+ gcc_jit_context_new_rvalue_from_int (
+ comp.ctxt,
+ comp.unsigned_type,
+ (Lisp_Int0 >> !USE_LSB_TAG)));
+
+ gcc_jit_rvalue *res =
+ gcc_jit_context_new_unary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_UNARY_OP_LOGICAL_NEGATE,
+ comp.int_type,
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_BITWISE_AND,
+ comp.unsigned_type,
+ minus_res,
+ gcc_jit_context_new_rvalue_from_int (
+ comp.ctxt,
+ comp.unsigned_type,
+ ((1 << INTTYPEBITS) - 1))));
+
+ return res;
+}
+
+static gcc_jit_rvalue *
+emit_XFIXNUM (gcc_jit_rvalue *obj)
+{
+ emit_comment ("XFIXNUM");
+
+ return gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_RSHIFT,
+ comp.emacs_int_type,
+ emit_XLI (obj),
+ comp.inttypebits);
+}
+
+static gcc_jit_rvalue *
+emit_INTEGERP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("INTEGERP");
+
+ return gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_OR,
+ comp.bool_type,
+ emit_cast (comp.bool_type,
+ emit_FIXNUMP (obj)),
+ emit_BIGNUMP (obj));
+}
+
+static gcc_jit_rvalue *
+emit_NUMBERP (gcc_jit_rvalue *obj)
+{
+ emit_comment ("NUMBERP");
+
+ return gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_OR,
+ comp.bool_type,
+ emit_INTEGERP (obj),
+ emit_cast (comp.bool_type,
+ emit_FLOATP (obj)));
+}
+
+static gcc_jit_rvalue *
+emit_make_fixnum (gcc_jit_rvalue *obj)
+{
+ emit_comment ("make_fixnum");
+
+ gcc_jit_rvalue *tmp =
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LSHIFT,
+ comp.emacs_int_type,
+ obj,
+ comp.inttypebits);
+
+ tmp = gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_PLUS,
+ comp.emacs_int_type,
+ tmp,
+ comp.lisp_int0);
+
+ gcc_jit_lvalue *res = gcc_jit_function_new_local (comp.func,
+ NULL,
+ comp.lisp_obj_type,
+ "lisp_obj_fixnum");
+
+ gcc_jit_block_add_assignment (comp.block,
+ NULL,
+ emit_lval_XLI (res),
+ tmp);
+
+ return gcc_jit_lvalue_as_rvalue (res);
+}
+
+static gcc_jit_rvalue *
+emit_const_lisp_obj (Lisp_Object obj)
+{
+ emit_comment (format_string ("const lisp obj: %s",
+ SSDATA (Fprin1_to_string (obj, Qnil))));
+
+ if (Qnil == NULL && EQ (obj, Qnil))
+ return emit_cast (comp.lisp_obj_type,
+ gcc_jit_context_new_rvalue_from_ptr (comp.ctxt,
+ comp.void_ptr_type,
+ NULL));
+
+ Lisp_Object d_reloc_idx = CALL1I (comp-ctxt-data-relocs-idx, Vcomp_ctxt);
+ ptrdiff_t reloc_fixn = XFIXNUM (Fgethash (obj, d_reloc_idx, Qnil));
+ gcc_jit_rvalue *reloc_n =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ reloc_fixn);
+ return
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_context_new_array_access (comp.ctxt,
+ NULL,
+ comp.data_relocs,
+ reloc_n));
+}
+
+static gcc_jit_rvalue *
+emit_NILP (gcc_jit_rvalue *x)
+{
+ emit_comment ("NILP");
+ return emit_EQ (x, emit_const_lisp_obj (Qnil));
+}
+
+static gcc_jit_rvalue *
+emit_XCAR (gcc_jit_rvalue *c)
+{
+ emit_comment ("XCAR");
+
+ /* XCONS (c)->u.s.car */
+ return
+ gcc_jit_rvalue_access_field (
+ /* XCONS (c)->u.s */
+ gcc_jit_rvalue_access_field (
+ /* XCONS (c)->u */
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference_field (
+ emit_XCONS (c),
+ NULL,
+ comp.lisp_cons_u)),
+ NULL,
+ comp.lisp_cons_u_s),
+ NULL,
+ comp.lisp_cons_u_s_car);
+}
+
+static gcc_jit_lvalue *
+emit_lval_XCAR (gcc_jit_rvalue *c)
+{
+ emit_comment ("lval_XCAR");
+
+ /* XCONS (c)->u.s.car */
+ return
+ gcc_jit_lvalue_access_field (
+ /* XCONS (c)->u.s */
+ gcc_jit_lvalue_access_field (
+ /* XCONS (c)->u */
+ gcc_jit_rvalue_dereference_field (
+ emit_XCONS (c),
+ NULL,
+ comp.lisp_cons_u),
+ NULL,
+ comp.lisp_cons_u_s),
+ NULL,
+ comp.lisp_cons_u_s_car);
+}
+
+static gcc_jit_rvalue *
+emit_XCDR (gcc_jit_rvalue *c)
+{
+ emit_comment ("XCDR");
+ /* XCONS (c)->u.s.u.cdr */
+ return
+ gcc_jit_rvalue_access_field (
+ /* XCONS (c)->u.s.u */
+ gcc_jit_rvalue_access_field (
+ /* XCONS (c)->u.s */
+ gcc_jit_rvalue_access_field (
+ /* XCONS (c)->u */
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference_field (
+ emit_XCONS (c),
+ NULL,
+ comp.lisp_cons_u)),
+ NULL,
+ comp.lisp_cons_u_s),
+ NULL,
+ comp.lisp_cons_u_s_u),
+ NULL,
+ comp.lisp_cons_u_s_u_cdr);
+}
+
+static gcc_jit_lvalue *
+emit_lval_XCDR (gcc_jit_rvalue *c)
+{
+ emit_comment ("lval_XCDR");
+
+ /* XCONS (c)->u.s.u.cdr */
+ return
+ gcc_jit_lvalue_access_field (
+ /* XCONS (c)->u.s.u */
+ gcc_jit_lvalue_access_field (
+ /* XCONS (c)->u.s */
+ gcc_jit_lvalue_access_field (
+ /* XCONS (c)->u */
+ gcc_jit_rvalue_dereference_field (
+ emit_XCONS (c),
+ NULL,
+ comp.lisp_cons_u),
+ NULL,
+ comp.lisp_cons_u_s),
+ NULL,
+ comp.lisp_cons_u_s_u),
+ NULL,
+ comp.lisp_cons_u_s_u_cdr);
+}
+
+static void
+emit_CHECK_CONS (gcc_jit_rvalue *x)
+{
+ emit_comment ("CHECK_CONS");
+
+ gcc_jit_rvalue *args[] =
+ { emit_CONSP (x),
+ emit_const_lisp_obj (Qconsp),
+ x };
+
+ gcc_jit_block_add_eval (
+ comp.block,
+ NULL,
+ gcc_jit_context_new_call (comp.ctxt,
+ NULL,
+ comp.check_type,
+ 3,
+ args));
+}
+
+static gcc_jit_rvalue *
+emit_car_addr (gcc_jit_rvalue *c)
+{
+ emit_comment ("car_addr");
+
+ return gcc_jit_lvalue_get_address (emit_lval_XCAR (c), NULL);
+}
+
+static gcc_jit_rvalue *
+emit_cdr_addr (gcc_jit_rvalue *c)
+{
+ emit_comment ("cdr_addr");
+
+ return gcc_jit_lvalue_get_address (emit_lval_XCDR (c), NULL);
+}
+
+static void
+emit_XSETCAR (gcc_jit_rvalue *c, gcc_jit_rvalue *n)
+{
+ emit_comment ("XSETCAR");
+
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ gcc_jit_rvalue_dereference (
+ emit_car_addr (c),
+ NULL),
+ n);
+}
+
+static void
+emit_XSETCDR (gcc_jit_rvalue *c, gcc_jit_rvalue *n)
+{
+ emit_comment ("XSETCDR");
+
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ gcc_jit_rvalue_dereference (
+ emit_cdr_addr (c),
+ NULL),
+ n);
+}
+
+static gcc_jit_rvalue *
+emit_PURE_P (gcc_jit_rvalue *ptr)
+{
+
+ emit_comment ("PURE_P");
+
+ return
+ gcc_jit_context_new_comparison (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_COMPARISON_LE,
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MINUS,
+ comp.uintptr_type,
+ emit_cast (comp.uintptr_type, ptr),
+ emit_cast (comp.uintptr_type,
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference (comp.pure_ref, NULL)))),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.uintptr_type,
+ PURESIZE));
+}
+
+\f
+/*************************************/
+/* Code emitted by LIMPLE statemes. */
+/*************************************/
+
+/* Emit an r-value from an mvar meta variable.
+ In case this is a constant that was propagated return it otherwise load it
+ from frame. */
+
+static gcc_jit_rvalue *
+emit_mvar_val (Lisp_Object mvar)
+{
+ Lisp_Object const_vld = CALL1I (comp-mvar-const-vld, mvar);
+ Lisp_Object constant = CALL1I (comp-mvar-constant, mvar);
+
+ if (!NILP (const_vld))
+ {
+ if (FIXNUMP (constant))
+ {
+ /* We can still emit directly objects that are self-contained in a
+ word (read fixnums). */
+ emit_comment (SSDATA (Fprin1_to_string (constant, Qnil)));
+ gcc_jit_rvalue *word =
+ gcc_jit_context_new_rvalue_from_ptr (comp.ctxt,
+ comp.void_ptr_type,
+ constant);
+ return emit_cast (comp.lisp_obj_type, word);
+ }
+ /* Other const objects are fetched from the reloc array. */
+ return emit_const_lisp_obj (constant);
+ }
+
+ return gcc_jit_lvalue_as_rvalue (get_slot (mvar));
+}
+
+static void
+emit_frame_assignment (Lisp_Object dst_mvar, gcc_jit_rvalue *val)
+{
+
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ get_slot (dst_mvar),
+ val);
+}
+
+static gcc_jit_rvalue *
+emit_set_internal (Lisp_Object args)
+{
+ /*
+ Ex: (set_internal #s(comp-mvar nil nil t comp-test-up-val nil nil)
+ #s(comp-mvar 1 4 t nil symbol nil)).
+ */
+ /* TODO: Inline the most common case. */
+ if (list_length (args) != 3)
+ xsignal2 (Qnative_ice,
+ build_string ("unexpected arg length for insns"),
+ args);
+
+ args = XCDR (args);
+ int i = 0;
+ gcc_jit_rvalue *gcc_args[4];
+ FOR_EACH_TAIL (args)
+ gcc_args[i++] = emit_mvar_val (XCAR (args));
+ gcc_args[2] = emit_const_lisp_obj (Qnil);
+ gcc_args[3] = gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.int_type,
+ SET_INTERNAL_SET);
+ return emit_call (intern_c_string ("set_internal"), comp.void_type , 4,
+ gcc_args, false);
+}
+
+/* This is for a regular function with arguments as m-var. */
+
+static gcc_jit_rvalue *
+emit_simple_limple_call (Lisp_Object args, gcc_jit_type *ret_type, bool direct)
+{
+ USE_SAFE_ALLOCA;
+ int i = 0;
+ Lisp_Object callee = FIRST (args);
+ args = XCDR (args);
+ ptrdiff_t nargs = list_length (args);
+ gcc_jit_rvalue **gcc_args = SAFE_ALLOCA (nargs * sizeof (*gcc_args));
+ FOR_EACH_TAIL (args)
+ gcc_args[i++] = emit_mvar_val (XCAR (args));
+
+ SAFE_FREE ();
+ return emit_call (callee, ret_type, nargs, gcc_args, direct);
+}
+
+static gcc_jit_rvalue *
+emit_simple_limple_call_lisp_ret (Lisp_Object args)
+{
+ /*
+ Ex: (call Fcons #s(comp-mvar 3 0 t 1 nil) #s(comp-mvar 4 nil t nil nil)).
+ */
+ return emit_simple_limple_call (args, comp.lisp_obj_type, false);
+}
+
+static gcc_jit_rvalue *
+emit_simple_limple_call_void_ret (Lisp_Object args)
+{
+ return emit_simple_limple_call (args, comp.void_type, false);
+}
+
+/* Entry point to dispatch emitting (call fun ...). */
+
+static gcc_jit_rvalue *
+emit_limple_call (Lisp_Object insn)
+{
+ Lisp_Object callee_sym = FIRST (insn);
+ Lisp_Object emitter = Fgethash (callee_sym, comp.emitter_dispatcher, Qnil);
+
+ if (!NILP (emitter))
+ {
+ gcc_jit_rvalue * (* emitter_ptr) (Lisp_Object) = xmint_pointer (emitter);
+ return emitter_ptr (insn);
+ }
+
+ return emit_simple_limple_call_lisp_ret (insn);
+}
+
+static gcc_jit_rvalue *
+emit_limple_call_ref (Lisp_Object insn, bool direct)
+{
+ /* Ex: (funcall #s(comp-mvar 1 5 t eql symbol t)
+ #s(comp-mvar 2 6 nil nil nil t)
+ #s(comp-mvar 3 7 t 0 fixnum t)). */
+
+ Lisp_Object callee = FIRST (insn);
+ EMACS_INT nargs = XFIXNUM (Flength (CDR (insn)));
+ EMACS_INT base_ptr = 0;
+ if (nargs)
+ base_ptr = XFIXNUM (CALL1I (comp-mvar-slot, SECOND (insn)));
+ return emit_call_ref (callee, nargs, comp.frame[base_ptr], direct);
+}
+
+/* Register an handler for a non local exit. */
+
+static void
+emit_limple_push_handler (gcc_jit_rvalue *handler, gcc_jit_rvalue *handler_type,
+ gcc_jit_block *handler_bb, gcc_jit_block *guarded_bb,
+ Lisp_Object clobbered_mvar)
+{
+ /* struct handler *c = push_handler (POP, type); */
+
+ gcc_jit_rvalue *args[] = { handler, handler_type };
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ comp.loc_handler,
+ emit_call (intern_c_string ("push_handler"),
+ comp.handler_ptr_type, 2, args, false));
+
+ args[0] =
+ gcc_jit_lvalue_get_address (
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (comp.loc_handler),
+ NULL,
+ comp.handler_jmp_field),
+ NULL);
+
+ gcc_jit_rvalue *res;
+ res =
+ emit_call (intern_c_string (SETJMP_NAME), comp.int_type, 1, args, false);
+ emit_cond_jump (res, handler_bb, guarded_bb);
+}
+
+static void
+emit_limple_insn (Lisp_Object insn)
+{
+ Lisp_Object op = XCAR (insn);
+ Lisp_Object args = XCDR (insn);
+ gcc_jit_rvalue *res;
+ Lisp_Object arg[6];
+
+ Lisp_Object p = XCDR (insn);
+ ptrdiff_t i = 0;
+ FOR_EACH_TAIL (p)
+ {
+ if (i == sizeof (arg) / sizeof (Lisp_Object))
+ break;
+ arg[i++] = XCAR (p);
+ }
+
+ if (EQ (op, Qjump))
+ {
+ /* Unconditional branch. */
+ gcc_jit_block *target = retrive_block (arg[0]);
+ gcc_jit_block_end_with_jump (comp.block, NULL, target);
+ }
+ else if (EQ (op, Qcond_jump))
+ {
+ /* Conditional branch. */
+ gcc_jit_rvalue *a = emit_mvar_val (arg[0]);
+ gcc_jit_rvalue *b = emit_mvar_val (arg[1]);
+ gcc_jit_block *target1 = retrive_block (arg[2]);
+ gcc_jit_block *target2 = retrive_block (arg[3]);
+
+ emit_cond_jump (emit_EQ (a, b), target2, target1);
+ }
+ else if (EQ (op, Qcond_jump_narg_leq))
+ {
+ /*
+ Limple: (cond-jump-narg-less 2 entry_2 entry_fallback_2)
+ C: if (nargs < 2) goto entry2_fallback; else goto entry_2;
+ */
+ gcc_jit_lvalue *nargs =
+ gcc_jit_param_as_lvalue (gcc_jit_function_get_param (comp.func, 0));
+ gcc_jit_rvalue *n =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ XFIXNUM (arg[0]));
+ gcc_jit_block *target1 = retrive_block (arg[1]);
+ gcc_jit_block *target2 = retrive_block (arg[2]);
+ gcc_jit_rvalue *test = gcc_jit_context_new_comparison (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_COMPARISON_LE,
+ gcc_jit_lvalue_as_rvalue (nargs),
+ n);
+ emit_cond_jump (test, target2, target1);
+ }
+ else if (EQ (op, Qphi))
+ {
+ /* Nothing to do for phis into the backend. */
+ }
+ else if (EQ (op, Qpush_handler))
+ {
+ /* (push-handler condition-case #s(comp-mvar 0 3 t (arith-error) cons nil) 1 bb_2 bb_1) */
+ int h_num UNINIT;
+ Lisp_Object handler_spec = arg[0];
+ gcc_jit_rvalue *handler = emit_mvar_val (arg[1]);
+ if (EQ (handler_spec, Qcatcher))
+ h_num = CATCHER;
+ else if (EQ (handler_spec, Qcondition_case))
+ h_num = CONDITION_CASE;
+ else
+ xsignal2 (Qnative_ice, build_string ("incoherent insn"), insn);
+ gcc_jit_rvalue *handler_type =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.int_type,
+ h_num);
+ gcc_jit_block *handler_bb = retrive_block (arg[2]);
+ gcc_jit_block *guarded_bb = retrive_block (arg[3]);
+ emit_limple_push_handler (handler, handler_type, handler_bb, guarded_bb,
+ arg[0]);
+ }
+ else if (EQ (op, Qpop_handler))
+ {
+ /*
+ C: current_thread->m_handlerlist =
+ current_thread->m_handlerlist->next;
+ */
+ gcc_jit_lvalue *m_handlerlist =
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference (comp.current_thread_ref, NULL)),
+ NULL,
+ comp.m_handlerlist);
+
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ m_handlerlist,
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (m_handlerlist),
+ NULL,
+ comp.handler_next_field)));
+
+ }
+ else if (EQ (op, Qfetch_handler))
+ {
+ gcc_jit_lvalue *m_handlerlist =
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference (comp.current_thread_ref, NULL)),
+ NULL,
+ comp.m_handlerlist);
+ gcc_jit_block_add_assignment (comp.block,
+ NULL,
+ comp.loc_handler,
+ gcc_jit_lvalue_as_rvalue (m_handlerlist));
+
+ gcc_jit_block_add_assignment (
+ comp.block,
+ NULL,
+ m_handlerlist,
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (comp.loc_handler),
+ NULL,
+ comp.handler_next_field)));
+ emit_frame_assignment (
+ arg[0],
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_rvalue_dereference_field (
+ gcc_jit_lvalue_as_rvalue (comp.loc_handler),
+ NULL,
+ comp.handler_val_field)));
+ }
+ else if (EQ (op, Qcall))
+ {
+ gcc_jit_block_add_eval (comp.block, NULL,
+ emit_limple_call (args));
+ }
+ else if (EQ (op, Qcallref))
+ {
+ gcc_jit_block_add_eval (comp.block, NULL,
+ emit_limple_call_ref (args, false));
+ }
+ else if (EQ (op, Qdirect_call))
+ {
+ gcc_jit_block_add_eval (
+ comp.block, NULL,
+ emit_simple_limple_call (XCDR (insn), comp.lisp_obj_type, true));
+ }
+ else if (EQ (op, Qdirect_callref))
+ {
+ gcc_jit_block_add_eval (comp.block, NULL,
+ emit_limple_call_ref (XCDR (insn), true));
+ }
+ else if (EQ (op, Qset))
+ {
+ Lisp_Object arg1 = arg[1];
+
+ if (EQ (Ftype_of (arg1), Qcomp_mvar))
+ res = emit_mvar_val (arg1);
+ else if (EQ (FIRST (arg1), Qcall))
+ res = emit_limple_call (XCDR (arg1));
+ else if (EQ (FIRST (arg1), Qcallref))
+ res = emit_limple_call_ref (XCDR (arg1), false);
+ else if (EQ (FIRST (arg1), Qdirect_call))
+ res = emit_simple_limple_call (XCDR (arg1), comp.lisp_obj_type, true);
+ else if (EQ (FIRST (arg1), Qdirect_callref))
+ res = emit_limple_call_ref (XCDR (arg1), true);
+ else
+ xsignal2 (Qnative_ice,
+ build_string ("LIMPLE inconsistent arg1 for insn"),
+ insn);
+
+ if (!res)
+ xsignal1 (Qnative_ice,
+ build_string (gcc_jit_context_get_first_error (comp.ctxt)));
+
+ emit_frame_assignment (arg[0], res);
+ }
+ else if (EQ (op, Qset_par_to_local))
+ {
+ /* Ex: (set-par-to-local #s(comp-mvar 0 3 nil nil nil nil) 0). */
+ EMACS_INT param_n = XFIXNUM (arg[1]);
+ gcc_jit_rvalue *param =
+ gcc_jit_param_as_rvalue (gcc_jit_function_get_param (comp.func,
+ param_n));
+ emit_frame_assignment (arg[0], param);
+ }
+ else if (EQ (op, Qset_args_to_local))
+ {
+ /*
+ Ex: (set-args-to-local #s(comp-mvar 1 6 nil nil nil nil))
+ C: local[1] = *args;
+ */
+ gcc_jit_rvalue *gcc_args =
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_param_as_lvalue (gcc_jit_function_get_param (comp.func, 1)));
+
+ gcc_jit_rvalue *res =
+ gcc_jit_lvalue_as_rvalue (gcc_jit_rvalue_dereference (gcc_args, NULL));
+
+ emit_frame_assignment (arg[0], res);
+ }
+ else if (EQ (op, Qset_rest_args_to_local))
+ {
+ /*
+ Ex: (set-rest-args-to-local #s(comp-mvar 2 9 nil nil nil nil))
+ C: local[2] = list (nargs - 2, args);
+ */
+
+ EMACS_INT slot_n = XFIXNUM (CALL1I (comp-mvar-slot, arg[0]));
+ gcc_jit_rvalue *n =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ slot_n);
+ gcc_jit_lvalue *nargs =
+ gcc_jit_param_as_lvalue (gcc_jit_function_get_param (comp.func, 0));
+ gcc_jit_lvalue *args =
+ gcc_jit_param_as_lvalue (gcc_jit_function_get_param (comp.func, 1));
+
+ gcc_jit_rvalue *list_args[] =
+ { gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_MINUS,
+ comp.ptrdiff_type,
+ gcc_jit_lvalue_as_rvalue (nargs),
+ n),
+ gcc_jit_lvalue_as_rvalue (args) };
+
+ res = emit_call (Qlist, comp.lisp_obj_type, 2,
+ list_args, false);
+
+ emit_frame_assignment (arg[0], res);
+ }
+ else if (EQ (op, Qinc_args))
+ {
+ /*
+ Ex: (inc-args)
+ C: ++args;
+ */
+ gcc_jit_lvalue *args =
+ gcc_jit_param_as_lvalue (gcc_jit_function_get_param (comp.func, 1));
+
+ gcc_jit_block_add_assignment (comp.block,
+ NULL,
+ args,
+ emit_ptr_arithmetic (
+ gcc_jit_lvalue_as_rvalue (args),
+ comp.lisp_obj_ptr_type,
+ sizeof (Lisp_Object),
+ comp.one));
+ }
+ else if (EQ (op, Qsetimm))
+ {
+ /* Ex: (setimm #s(comp-mvar 9 1 t 3 nil) 3 a). */
+ gcc_jit_rvalue *reloc_n =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.int_type,
+ XFIXNUM (arg[1]));
+ emit_comment (SSDATA (Fprin1_to_string (arg[2], Qnil)));
+ emit_frame_assignment (
+ arg[0],
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_context_new_array_access (comp.ctxt,
+ NULL,
+ comp.data_relocs,
+ reloc_n)));
+ }
+ else if (EQ (op, Qcomment))
+ {
+ /* Ex: (comment "Function: foo"). */
+ emit_comment (SSDATA (arg[0]));
+ }
+ else if (EQ (op, Qreturn))
+ {
+ gcc_jit_block_end_with_return (comp.block,
+ NULL,
+ emit_mvar_val (arg[0]));
+ }
+ else
+ {
+ xsignal2 (Qnative_ice,
+ build_string ("LIMPLE op inconsistent"),
+ op);
+ }
+}
+
+\f
+/**************/
+/* Inliners. */
+/**************/
+
+static gcc_jit_rvalue *
+emit_call_with_type_hint (gcc_jit_function *func, Lisp_Object insn,
+ Lisp_Object type)
+{
+ bool type_hint = EQ (CALL1I (comp-mvar-type, SECOND (insn)), type);
+ gcc_jit_rvalue *args[] =
+ { emit_mvar_val (SECOND (insn)),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.bool_type,
+ type_hint) };
+
+ return gcc_jit_context_new_call (comp.ctxt, NULL, func, 2, args);
+}
+
+/* Same as before but with two args. The type hint is on the 2th. */
+static gcc_jit_rvalue *
+emit_call2_with_type_hint (gcc_jit_function *func, Lisp_Object insn,
+ Lisp_Object type)
+{
+ bool type_hint = EQ (CALL1I (comp-mvar-type, SECOND (insn)), type);
+ gcc_jit_rvalue *args[] =
+ { emit_mvar_val (SECOND (insn)),
+ emit_mvar_val (THIRD (insn)),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.bool_type,
+ type_hint) };
+
+ return gcc_jit_context_new_call (comp.ctxt, NULL, func, 3, args);
+}
+
+
+static gcc_jit_rvalue *
+emit_add1 (Lisp_Object insn)
+{
+ return emit_call_with_type_hint (comp.add1, insn, Qfixnum);
+}
+
+static gcc_jit_rvalue *
+emit_sub1 (Lisp_Object insn)
+{
+ return emit_call_with_type_hint (comp.sub1, insn, Qfixnum);
+}
+
+static gcc_jit_rvalue *
+emit_negate (Lisp_Object insn)
+{
+ return emit_call_with_type_hint (comp.negate, insn, Qfixnum);
+}
+
+static gcc_jit_rvalue *
+emit_consp (Lisp_Object insn)
+{
+ gcc_jit_rvalue *x = emit_mvar_val (SECOND (insn));
+ gcc_jit_rvalue *res = emit_cast (comp.bool_type,
+ emit_CONSP (x));
+ return gcc_jit_context_new_call (comp.ctxt,
+ NULL,
+ comp.bool_to_lisp_obj,
+ 1, &res);
+}
+
+static gcc_jit_rvalue *
+emit_car (Lisp_Object insn)
+{
+ return emit_call_with_type_hint (comp.car, insn, Qcons);
+}
+
+static gcc_jit_rvalue *
+emit_cdr (Lisp_Object insn)
+{
+ return emit_call_with_type_hint (comp.cdr, insn, Qcons);
+}
+
+static gcc_jit_rvalue *
+emit_setcar (Lisp_Object insn)
+{
+ return emit_call2_with_type_hint (comp.setcar, insn, Qcons);
+}
+
+static gcc_jit_rvalue *
+emit_setcdr (Lisp_Object insn)
+{
+ return emit_call2_with_type_hint (comp.setcdr, insn, Qcons);
+}
+
+static gcc_jit_rvalue *
+emit_numperp (Lisp_Object insn)
+{
+ gcc_jit_rvalue *x = emit_mvar_val (SECOND (insn));
+ gcc_jit_rvalue *res = emit_NUMBERP (x);
+ return gcc_jit_context_new_call (comp.ctxt, NULL, comp.bool_to_lisp_obj, 1,
+ &res);
+}
+
+static gcc_jit_rvalue *
+emit_integerp (Lisp_Object insn)
+{
+ gcc_jit_rvalue *x = emit_mvar_val (SECOND (insn));
+ gcc_jit_rvalue *res = emit_INTEGERP (x);
+ return gcc_jit_context_new_call (comp.ctxt, NULL, comp.bool_to_lisp_obj, 1,
+ &res);
+}
+
+/* This is in charge of serializing an object and export a function to
+ retrieve it at load time. */
+static void
+emit_static_object (const char *name, Lisp_Object obj)
+{
+ /* libgccjit has no support for initialized static data.
+ The mechanism below is certainly not aesthetic but I assume the bottle neck
+ in terms of performance at load time will still be the reader.
+ NOTE: we can not relay on libgccjit even for valid NULL terminated C
+ strings cause of this funny bug that will affect all pre gcc10 era gccs:
+ https://gcc.gnu.org/ml/jit/2019-q3/msg00013.html */
+
+ Lisp_Object str = Fprin1_to_string (obj, Qnil);
+ ptrdiff_t len = SBYTES (str);
+ const char *p = SSDATA (str);
+
+ gcc_jit_type *a_type =
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ len + 1);
+ gcc_jit_field *fields[] =
+ { gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.ptrdiff_type,
+ "len"),
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ a_type,
+ "data") };
+
+ gcc_jit_type *data_struct_t =
+ gcc_jit_struct_as_type (
+ gcc_jit_context_new_struct_type (comp.ctxt,
+ NULL,
+ format_string ("%s_struct", name),
+ 2, fields));
+
+ gcc_jit_lvalue *data_struct =
+ gcc_jit_context_new_global (comp.ctxt,
+ NULL,
+ GCC_JIT_GLOBAL_INTERNAL,
+ data_struct_t,
+ format_string ("%s_s", name));
+
+ gcc_jit_function *f =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_EXPORTED,
+ gcc_jit_type_get_pointer (data_struct_t),
+ name,
+ 0, NULL, 0);
+ DECL_BLOCK (block, f);
+
+ /* NOTE this truncates if the data has some zero byte before termination. */
+ gcc_jit_block_add_comment (block, NULL, p);
+
+ gcc_jit_lvalue *arr =
+ gcc_jit_lvalue_access_field (data_struct, NULL, fields[1]);
+
+ for (ptrdiff_t i = 0; i < len; i++, p++)
+ {
+ gcc_jit_block_add_assignment (
+ block,
+ NULL,
+ gcc_jit_context_new_array_access (
+ comp.ctxt,
+ NULL,
+ gcc_jit_lvalue_as_rvalue (arr),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ i)),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.char_type,
+ *p));
+ }
+ gcc_jit_block_add_assignment (
+ block,
+ NULL,
+ gcc_jit_lvalue_access_field (data_struct, NULL, fields[0]),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.ptrdiff_type,
+ len));
+ gcc_jit_rvalue *res = gcc_jit_lvalue_get_address (data_struct, NULL);
+ gcc_jit_block_end_with_return (block, NULL, res);
+}
+
+static void
+declare_runtime_imported_data (void)
+{
+ /* Imported symbols by inliner functions. */
+ CALL1I (comp-add-const-to-relocs, Qnil);
+ CALL1I (comp-add-const-to-relocs, Qt);
+ CALL1I (comp-add-const-to-relocs, Qconsp);
+ CALL1I (comp-add-const-to-relocs, Qlistp);
+}
+
+/*
+ Declare as imported all the functions that are requested from the runtime.
+ These are either subrs or not.
+*/
+static Lisp_Object
+declare_runtime_imported_funcs (void)
+{
+ /* For subr imported by the runtime we rely on the standard mechanism in place
+ for functions imported by lisp code. */
+ CALL1I (comp-add-subr-to-relocs, intern_c_string ("1+"));
+ CALL1I (comp-add-subr-to-relocs, intern_c_string ("1-"));
+ CALL1I (comp-add-subr-to-relocs, Qplus);
+ CALL1I (comp-add-subr-to-relocs, Qminus);
+ CALL1I (comp-add-subr-to-relocs, Qlist);
+
+ Lisp_Object field_list = Qnil;
+#define ADD_IMPORTED(f_name, ret_type, nargs, args) \
+ { \
+ Lisp_Object name = intern_c_string (f_name); \
+ Lisp_Object field = \
+ make_mint_ptr (declare_imported_func (name, ret_type, nargs, args)); \
+ Lisp_Object el = Fcons (name, field); \
+ field_list = Fcons (el, field_list); \
+ } while (0)
+
+ gcc_jit_type *args[4];
+
+ ADD_IMPORTED ("wrong_type_argument", comp.void_type, 2, NULL);
+
+ args[0] = comp.lisp_obj_type;
+ args[1] = comp.int_type;
+ ADD_IMPORTED ("helper_PSEUDOVECTOR_TYPEP_XUNTAG", comp.bool_type, 2, args);
+
+ ADD_IMPORTED ("pure_write_error", comp.void_type, 1, NULL);
+
+ args[0] = comp.lisp_obj_type;
+ args[1] = comp.int_type;
+ ADD_IMPORTED ("push_handler", comp.handler_ptr_type, 2, args);
+
+ args[0] = gcc_jit_type_get_pointer (gcc_jit_struct_as_type (comp.jmp_buf_s));
+ ADD_IMPORTED (SETJMP_NAME, comp.int_type, 1, args);
+
+ ADD_IMPORTED ("record_unwind_protect_excursion", comp.void_type, 0, NULL);
+
+ args[0] = comp.lisp_obj_type;
+ ADD_IMPORTED ("helper_unbind_n", comp.lisp_obj_type, 1, args);
+
+ ADD_IMPORTED ("helper_save_restriction", comp.void_type, 0, NULL);
+
+ ADD_IMPORTED ("record_unwind_current_buffer", comp.void_type, 0, NULL);
+
+ args[0] = args[1] = args[2] = comp.lisp_obj_type;
+ args[3] = comp.int_type;
+ ADD_IMPORTED ("set_internal", comp.void_type, 4, args);
+
+ args[0] = comp.lisp_obj_type;
+ ADD_IMPORTED ("helper_unwind_protect", comp.void_type, 1, args);
+
+ args[0] = args[1] = comp.lisp_obj_type;
+ ADD_IMPORTED ("specbind", comp.void_type, 2, args);
+
+#undef ADD_IMPORTED
+
+ return field_list;
+}
+
+/*
+ This emit the code needed by every compilation unit to be loaded.
+*/
+static void
+emit_ctxt_code (void)
+{
+ USE_SAFE_ALLOCA;
+
+ comp.current_thread_ref =
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_context_new_global (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_GLOBAL_EXPORTED,
+ gcc_jit_type_get_pointer (comp.thread_state_ptr_type),
+ CURRENT_THREAD_RELOC_SYM));
+
+ comp.pure_ref =
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_context_new_global (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_GLOBAL_EXPORTED,
+ gcc_jit_type_get_pointer (comp.void_ptr_type),
+ PURE_RELOC_SYM));
+
+ declare_runtime_imported_data ();
+ /* Imported objects. */
+ EMACS_INT d_reloc_len =
+ XFIXNUM (CALL1I (hash-table-count,
+ CALL1I (comp-ctxt-data-relocs-idx, Vcomp_ctxt)));
+ Lisp_Object d_reloc = Fnreverse (CALL1I (comp-ctxt-data-relocs-l, Vcomp_ctxt));
+ d_reloc = Fvconcat (1, &d_reloc);
+
+ comp.data_relocs =
+ gcc_jit_lvalue_as_rvalue (
+ gcc_jit_context_new_global (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_GLOBAL_EXPORTED,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ d_reloc_len),
+ DATA_RELOC_SYM));
+
+ emit_static_object (TEXT_DATA_RELOC_SYM, d_reloc);
+
+ /* Imported functions from non Lisp code. */
+ Lisp_Object f_runtime = declare_runtime_imported_funcs ();
+ EMACS_INT f_reloc_len = XFIXNUM (Flength (f_runtime));
+
+ /* Imported subrs. */
+ Lisp_Object f_subr = CALL1I (comp-ctxt-func-relocs-l, Vcomp_ctxt);
+ f_reloc_len += XFIXNUM (Flength (f_subr));
+
+ gcc_jit_field **fields = SAFE_ALLOCA (f_reloc_len * sizeof (*fields));
+ Lisp_Object f_reloc_list = Qnil;
+ int n_frelocs = 0;
+
+ FOR_EACH_TAIL (f_runtime)
+ {
+ Lisp_Object el = XCAR (f_runtime);
+ fields[n_frelocs++] = xmint_pointer (XCDR (el));
+ f_reloc_list = Fcons (XCAR (el), f_reloc_list);
+ }
+
+ FOR_EACH_TAIL (f_subr)
+ {
+ Lisp_Object subr_sym = XCAR (f_subr);
+ Lisp_Object subr = symbol_subr (subr_sym);
+ /* Ignore inliners. This are not real functions to be imported. */
+ if (SUBRP (subr))
+ {
+ Lisp_Object maxarg = XCDR (Fsubr_arity (subr));
+ gcc_jit_field *field =
+ declare_imported_func (subr_sym, comp.lisp_obj_type,
+ FIXNUMP (maxarg) ? XFIXNUM (maxarg) :
+ EQ (maxarg, Qmany) ? MANY : UNEVALLED,
+ NULL);
+ fields[n_frelocs++] = field;
+ f_reloc_list = Fcons (subr_sym, f_reloc_list);
+ }
+ }
+
+ Lisp_Object f_reloc_vec = make_vector (n_frelocs, Qnil);
+ f_reloc_list = Fnreverse (f_reloc_list);
+ ptrdiff_t i = 0;
+ FOR_EACH_TAIL (f_reloc_list)
+ {
+ ASET (f_reloc_vec, i++, XCAR (f_reloc_list));
+ }
+ emit_static_object (TEXT_IMPORTED_FUNC_RELOC_SYM, f_reloc_vec);
+
+ gcc_jit_struct *f_reloc_struct =
+ gcc_jit_context_new_struct_type (comp.ctxt,
+ NULL,
+ "function_reloc_struct",
+ n_frelocs, fields);
+ comp.func_relocs =
+ gcc_jit_context_new_global (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_GLOBAL_EXPORTED,
+ gcc_jit_struct_as_type (f_reloc_struct),
+ IMPORTED_FUNC_RELOC_SYM);
+
+ SAFE_FREE ();
+}
+
+\f
+/****************************************************************/
+/* Inline function definition and lisp data structure follows. */
+/****************************************************************/
+
+/* struct Lisp_Cons definition. */
+
+static void
+define_lisp_cons (void)
+{
+ /*
+ union cdr_u
+ {
+ Lisp_Object cdr;
+ struct Lisp_Cons *chain;
+ };
+
+ struct cons_s
+ {
+ Lisp_Object car;
+ union cdr_u u;
+ };
+
+ union cons_u
+ {
+ struct cons_s s;
+ char align_pad[sizeof (struct Lisp_Cons)];
+ };
+
+ struct Lisp_Cons
+ {
+ union cons_u u;
+ };
+ */
+
+ comp.lisp_cons_s =
+ gcc_jit_context_new_opaque_struct (comp.ctxt,
+ NULL,
+ "comp_Lisp_Cons");
+ comp.lisp_cons_type =
+ gcc_jit_struct_as_type (comp.lisp_cons_s);
+ comp.lisp_cons_ptr_type =
+ gcc_jit_type_get_pointer (comp.lisp_cons_type);
+
+ comp.lisp_cons_u_s_u_cdr =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "cdr");
+
+ gcc_jit_field *cdr_u_fields[] =
+ { comp.lisp_cons_u_s_u_cdr,
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_cons_ptr_type,
+ "chain") };
+
+ gcc_jit_type *cdr_u =
+ gcc_jit_context_new_union_type (comp.ctxt,
+ NULL,
+ "comp_cdr_u",
+ sizeof (cdr_u_fields)
+ / sizeof (*cdr_u_fields),
+ cdr_u_fields);
+
+ comp.lisp_cons_u_s_car = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "car");
+ comp.lisp_cons_u_s_u = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ cdr_u,
+ "u");
+ gcc_jit_field *cons_s_fields[] =
+ { comp.lisp_cons_u_s_car,
+ comp.lisp_cons_u_s_u };
+
+ gcc_jit_struct *cons_s =
+ gcc_jit_context_new_struct_type (comp.ctxt,
+ NULL,
+ "comp_cons_s",
+ sizeof (cons_s_fields)
+ / sizeof (*cons_s_fields),
+ cons_s_fields);
+
+ comp.lisp_cons_u_s = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ gcc_jit_struct_as_type (cons_s),
+ "s");
+
+ gcc_jit_field *cons_u_fields[] =
+ { comp.lisp_cons_u_s,
+ gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ sizeof (struct Lisp_Cons)),
+ "align_pad") };
+
+ gcc_jit_type *lisp_cons_u_type =
+ gcc_jit_context_new_union_type (comp.ctxt,
+ NULL,
+ "comp_cons_u",
+ sizeof (cons_u_fields)
+ / sizeof (*cons_u_fields),
+ cons_u_fields);
+
+ comp.lisp_cons_u =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ lisp_cons_u_type,
+ "u");
+ gcc_jit_struct_set_fields (comp.lisp_cons_s,
+ NULL, 1, &comp.lisp_cons_u);
+
+}
+
+/* Opaque jmp_buf definition. */
+
+static void
+define_jmp_buf (void)
+{
+ gcc_jit_field *field =
+ gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ sizeof (jmp_buf)),
+ "stuff");
+ comp.jmp_buf_s =
+ gcc_jit_context_new_struct_type (comp.ctxt,
+ NULL,
+ "comp_jmp_buf",
+ 1, &field);
+}
+
+/* struct handler definition */
+
+static void
+define_handler_struct (void)
+{
+ comp.handler_s =
+ gcc_jit_context_new_opaque_struct (comp.ctxt, NULL, "comp_handler");
+ comp.handler_ptr_type =
+ gcc_jit_type_get_pointer (gcc_jit_struct_as_type (comp.handler_s));
+
+ comp.handler_jmp_field = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ gcc_jit_struct_as_type (
+ comp.jmp_buf_s),
+ "jmp");
+ comp.handler_val_field = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "val");
+ comp.handler_next_field = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.handler_ptr_type,
+ "next");
+ gcc_jit_field *fields[] =
+ { gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ offsetof (struct handler, val)),
+ "pad0"),
+ comp.handler_val_field,
+ comp.handler_next_field,
+ gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ offsetof (struct handler, jmp)
+ - offsetof (struct handler, next)
+ - sizeof (((struct handler *) 0)->next)),
+ "pad1"),
+ comp.handler_jmp_field,
+ gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ sizeof (struct handler)
+ - offsetof (struct handler, jmp)
+ - sizeof (((struct handler *) 0)->jmp)),
+ "pad2") };
+ gcc_jit_struct_set_fields (comp.handler_s,
+ NULL,
+ sizeof (fields) / sizeof (*fields),
+ fields);
+
+}
+
+static void
+define_thread_state_struct (void)
+{
+ /* Partially opaque definition for `thread_state'.
+ Because we need to access just m_handlerlist hopefully this is requires
+ less manutention then the full deifnition. */
+
+ comp.m_handlerlist = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.handler_ptr_type,
+ "m_handlerlist");
+ gcc_jit_field *fields[] =
+ { gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.char_type,
+ offsetof (struct thread_state,
+ m_handlerlist)),
+ "pad0"),
+ comp.m_handlerlist,
+ gcc_jit_context_new_field (
+ comp.ctxt,
+ NULL,
+ gcc_jit_context_new_array_type (
+ comp.ctxt,
+ NULL,
+ comp.char_type,
+ sizeof (struct thread_state)
+ - offsetof (struct thread_state,
+ m_handlerlist)
+ - sizeof (((struct thread_state *) 0)->m_handlerlist)),
+ "pad1") };
+
+ comp.thread_state_s =
+ gcc_jit_context_new_struct_type (comp.ctxt,
+ NULL,
+ "comp_thread_state",
+ sizeof (fields) / sizeof (*fields),
+ fields);
+ comp.thread_state_ptr_type =
+ gcc_jit_type_get_pointer (gcc_jit_struct_as_type (comp.thread_state_s));
+}
+
+static void
+define_cast_union (void)
+{
+
+ comp.cast_union_as_ll =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.long_long_type,
+ "ll");
+ comp.cast_union_as_ull =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.unsigned_long_long_type,
+ "ull");
+ comp.cast_union_as_l =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.long_type,
+ "l");
+ comp.cast_union_as_ul =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.unsigned_long_type,
+ "ul");
+ comp.cast_union_as_u =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.unsigned_type,
+ "u");
+ comp.cast_union_as_i =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.int_type,
+ "i");
+ comp.cast_union_as_b =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "b");
+ comp.cast_union_as_uintptr =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.uintptr_type,
+ "uintptr");
+ comp.cast_union_as_ptrdiff =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.ptrdiff_type,
+ "ptrdiff");
+ comp.cast_union_as_c_p =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.char_ptr_type,
+ "c_p");
+ comp.cast_union_as_v_p =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.void_ptr_type,
+ "v_p");
+ comp.cast_union_as_lisp_cons_ptr =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_cons_ptr_type,
+ "cons_ptr");
+ comp.cast_union_as_lisp_obj =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "lisp_obj");
+ comp.cast_union_as_lisp_obj_ptr =
+ gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.lisp_obj_ptr_type,
+ "lisp_obj_ptr");
+
+
+ gcc_jit_field *cast_union_fields[] =
+ { comp.cast_union_as_ll,
+ comp.cast_union_as_ull,
+ comp.cast_union_as_l,
+ comp.cast_union_as_ul,
+ comp.cast_union_as_u,
+ comp.cast_union_as_i,
+ comp.cast_union_as_b,
+ comp.cast_union_as_uintptr,
+ comp.cast_union_as_ptrdiff,
+ comp.cast_union_as_c_p,
+ comp.cast_union_as_v_p,
+ comp.cast_union_as_lisp_cons_ptr,
+ comp.cast_union_as_lisp_obj,
+ comp.cast_union_as_lisp_obj_ptr };
+ comp.cast_union_type =
+ gcc_jit_context_new_union_type (comp.ctxt,
+ NULL,
+ "cast_union",
+ sizeof (cast_union_fields)
+ / sizeof (*cast_union_fields),
+ cast_union_fields);
+}
+
+static void
+define_CHECK_TYPE (void)
+{
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.int_type,
+ "ok"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "predicate"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "x") };
+ comp.check_type =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_ALWAYS_INLINE,
+ comp.void_type,
+ "CHECK_TYPE",
+ 3,
+ param,
+ 0);
+ gcc_jit_rvalue *ok = gcc_jit_param_as_rvalue (param[0]);
+ gcc_jit_rvalue *predicate = gcc_jit_param_as_rvalue (param[1]);
+ gcc_jit_rvalue *x = gcc_jit_param_as_rvalue (param[2]);
+
+ DECL_BLOCK (entry_block, comp.check_type);
+ DECL_BLOCK (ok_block, comp.check_type);
+ DECL_BLOCK (not_ok_block, comp.check_type);
+
+ comp.block = entry_block;
+ comp.func = comp.check_type;
+
+ emit_cond_jump (ok, ok_block, not_ok_block);
+
+ gcc_jit_block_end_with_void_return (ok_block, NULL);
+
+ comp.block = not_ok_block;
+
+ gcc_jit_rvalue *wrong_type_args[] = { predicate, x };
+
+ gcc_jit_block_add_eval (comp.block,
+ NULL,
+ emit_call (intern_c_string ("wrong_type_argument"),
+ comp.void_type, 2, wrong_type_args,
+ false));
+
+ gcc_jit_block_end_with_void_return (not_ok_block, NULL);
+}
+
+/* Define a substitute for CAR as always inlined function. */
+
+static void
+define_CAR_CDR (void)
+{
+ gcc_jit_function *func[2];
+ char const *f_name[] = { "CAR", "CDR" };
+ for (int i = 0; i < 2; i++)
+ {
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "c"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "cert_cons") };
+ /* TODO: understand why after ipa-prop pass gcc is less keen on inlining
+ and as consequence can refuse to compile these. (see dhrystone.el)
+ Flag this and all the one involved in ipa-prop as
+ GCC_JIT_FUNCTION_INTERNAL not to fail compilation in case.
+ This seems at least to have no perf downside. */
+ func[i] =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_INTERNAL,
+ comp.lisp_obj_type,
+ f_name[i],
+ 2, param, 0);
+
+ gcc_jit_rvalue *c = gcc_jit_param_as_rvalue (param[0]);
+ DECL_BLOCK (entry_block, func[i]);
+ DECL_BLOCK (is_cons_b, func[i]);
+ DECL_BLOCK (not_a_cons_b, func[i]);
+ comp.block = entry_block;
+ comp.func = func[i];
+ emit_cond_jump (
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_OR,
+ comp.bool_type,
+ gcc_jit_param_as_rvalue (param[1]),
+ emit_cast (comp.bool_type,
+ emit_CONSP (c))),
+ is_cons_b,
+ not_a_cons_b);
+ comp.block = is_cons_b;
+ if (i == 0)
+ gcc_jit_block_end_with_return (comp.block, NULL, emit_XCAR (c));
+ else
+ gcc_jit_block_end_with_return (comp.block, NULL, emit_XCDR (c));
+
+ comp.block = not_a_cons_b;
+
+ DECL_BLOCK (is_nil_b, func[i]);
+ DECL_BLOCK (not_nil_b, func[i]);
+
+ emit_cond_jump (emit_NILP (c), is_nil_b, not_nil_b);
+
+ comp.block = is_nil_b;
+ gcc_jit_block_end_with_return (comp.block,
+ NULL,
+ emit_const_lisp_obj (Qnil));
+
+ comp.block = not_nil_b;
+ gcc_jit_rvalue *wrong_type_args[] =
+ { emit_const_lisp_obj (Qlistp), c };
+
+ gcc_jit_block_add_eval (comp.block,
+ NULL,
+ emit_call (intern_c_string ("wrong_type_argument"),
+ comp.void_type, 2, wrong_type_args,
+ false));
+ gcc_jit_block_end_with_return (comp.block,
+ NULL,
+ emit_const_lisp_obj (Qnil));
+ }
+ comp.car = func[0];
+ comp.cdr = func[1];
+}
+
+static void
+define_setcar_setcdr (void)
+{
+ char const *f_name[] = { "setcar", "setcdr" };
+ char const *par_name[] = { "new_car", "new_cdr" };
+
+ for (int i = 0; i < 2; i++)
+ {
+ gcc_jit_param *cell =
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "cell");
+ gcc_jit_param *new_el =
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ par_name[i]);
+
+ gcc_jit_param *param[] =
+ { cell,
+ new_el,
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "cert_cons") };
+
+ gcc_jit_function **f_ref = !i ? &comp.setcar : &comp.setcdr;
+ *f_ref = gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_INTERNAL,
+ comp.lisp_obj_type,
+ f_name[i],
+ 3, param, 0);
+ DECL_BLOCK (entry_block, *f_ref);
+ comp.func = *f_ref;
+ comp.block = entry_block;
+
+ /* CHECK_CONS (cell); */
+ emit_CHECK_CONS (gcc_jit_param_as_rvalue (cell));
+
+ /* CHECK_IMPURE (cell, XCONS (cell)); */
+ gcc_jit_rvalue *args[] =
+ { gcc_jit_param_as_rvalue (cell),
+ emit_XCONS (gcc_jit_param_as_rvalue (cell)) };
+
+ gcc_jit_block_add_eval (entry_block,
+ NULL,
+ gcc_jit_context_new_call (comp.ctxt,
+ NULL,
+ comp.check_impure,
+ 2,
+ args));
+
+ /* XSETCDR (cell, newel); */
+ if (!i)
+ emit_XSETCAR (gcc_jit_param_as_rvalue (cell),
+ gcc_jit_param_as_rvalue (new_el));
+ else
+ emit_XSETCDR (gcc_jit_param_as_rvalue (cell),
+ gcc_jit_param_as_rvalue (new_el));
+
+ /* return newel; */
+ gcc_jit_block_end_with_return (entry_block,
+ NULL,
+ gcc_jit_param_as_rvalue (new_el));
+ }
+}
+
+/*
+ Define a substitute for Fadd1 Fsub1.
+ Currently expose just fixnum arithmetic.
+*/
+
+static void
+define_add1_sub1 (void)
+{
+ gcc_jit_block *bb_orig = comp.block;
+ gcc_jit_function *func[2];
+ char const *f_name[] = { "add1", "sub1" };
+ char const *fall_back_func[] = { "1+", "1-" };
+ gcc_jit_rvalue *compare[] =
+ { comp.most_positive_fixnum, comp.most_negative_fixnum };
+ enum gcc_jit_binary_op op[] =
+ { GCC_JIT_BINARY_OP_PLUS, GCC_JIT_BINARY_OP_MINUS };
+ for (ptrdiff_t i = 0; i < 2; i++)
+ {
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "n"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "cert_fixnum") };
+ comp.func = func[i] =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_INTERNAL,
+ comp.lisp_obj_type,
+ f_name[i],
+ 2,
+ param, 0);
+ DECL_BLOCK (entry_block, func[i]);
+ DECL_BLOCK (inline_block, func[i]);
+ DECL_BLOCK (fcall_block, func[i]);
+
+ comp.block = entry_block;
+
+ /* cert_fixnum ||
+ ((FIXNUMP (n) && XFIXNUM (n) != MOST_POSITIVE_FIXNUM
+ ? (XFIXNUM (n) + 1)
+ : Fadd1 (n)) */
+
+ gcc_jit_rvalue *n = gcc_jit_param_as_rvalue (param[0]);
+ gcc_jit_rvalue *n_fixnum = emit_XFIXNUM (n);
+ gcc_jit_rvalue *sure_fixnum =
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_OR,
+ comp.bool_type,
+ gcc_jit_param_as_rvalue (param[1]),
+ emit_cast (comp.bool_type,
+ emit_FIXNUMP (n)));
+
+ emit_cond_jump (
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_AND,
+ comp.bool_type,
+ sure_fixnum,
+ gcc_jit_context_new_comparison (comp.ctxt,
+ NULL,
+ GCC_JIT_COMPARISON_NE,
+ n_fixnum,
+ compare[i])),
+ inline_block,
+ fcall_block);
+
+ comp.block = inline_block;
+ gcc_jit_rvalue *inline_res =
+ gcc_jit_context_new_binary_op (comp.ctxt,
+ NULL,
+ op[i],
+ comp.emacs_int_type,
+ n_fixnum,
+ comp.one);
+
+ gcc_jit_block_end_with_return (inline_block,
+ NULL,
+ emit_make_fixnum (inline_res));
+
+ comp.block = fcall_block;
+ gcc_jit_rvalue *call_res = emit_call (intern_c_string (fall_back_func[i]),
+ comp.lisp_obj_type, 1, &n, false);
+ gcc_jit_block_end_with_return (fcall_block,
+ NULL,
+ call_res);
+ }
+ comp.block = bb_orig;
+ comp.add1 = func[0];
+ comp.sub1 = func[1];
+}
+
+static void
+define_negate (void)
+{
+ gcc_jit_block *bb_orig = comp.block;
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "n"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "cert_fixnum") };
+
+ comp.func = comp.negate =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_INTERNAL,
+ comp.lisp_obj_type,
+ "negate",
+ 2, param, 0);
+
+ DECL_BLOCK (entry_block, comp.negate);
+ DECL_BLOCK (inline_block, comp.negate);
+ DECL_BLOCK (fcall_block, comp.negate);
+
+ comp.block = entry_block;
+
+ /* (cert_fixnum || FIXNUMP (TOP)) && XFIXNUM (TOP) != MOST_NEGATIVE_FIXNUM
+ ? make_fixnum (- XFIXNUM (TOP)) : Fminus (1, &TOP)) */
+
+ gcc_jit_lvalue *n = gcc_jit_param_as_lvalue (param[0]);
+ gcc_jit_rvalue *n_fixnum = emit_XFIXNUM (gcc_jit_lvalue_as_rvalue (n));
+ gcc_jit_rvalue *sure_fixnum =
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_OR,
+ comp.bool_type,
+ gcc_jit_param_as_rvalue (param[1]),
+ emit_cast (comp.bool_type,
+ emit_FIXNUMP (gcc_jit_lvalue_as_rvalue (n))));
+
+ emit_cond_jump (
+ gcc_jit_context_new_binary_op (
+ comp.ctxt,
+ NULL,
+ GCC_JIT_BINARY_OP_LOGICAL_AND,
+ comp.bool_type,
+ sure_fixnum,
+ gcc_jit_context_new_comparison (comp.ctxt,
+ NULL,
+ GCC_JIT_COMPARISON_NE,
+ n_fixnum,
+ comp.most_negative_fixnum)),
+ inline_block,
+ fcall_block);
+
+ comp.block = inline_block;
+ gcc_jit_rvalue *inline_res =
+ gcc_jit_context_new_unary_op (comp.ctxt,
+ NULL,
+ GCC_JIT_UNARY_OP_MINUS,
+ comp.emacs_int_type,
+ n_fixnum);
+
+ gcc_jit_block_end_with_return (inline_block,
+ NULL,
+ emit_make_fixnum (inline_res));
+
+ comp.block = fcall_block;
+ gcc_jit_rvalue *call_res = emit_call_ref (Qminus, 1, n, false);
+ gcc_jit_block_end_with_return (fcall_block,
+ NULL,
+ call_res);
+ comp.block = bb_orig;
+}
+
+/* Define a substitute for PSEUDOVECTORP as always inlined function. */
+
+static void
+define_PSEUDOVECTORP (void)
+{
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "a"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.int_type,
+ "code") };
+
+ comp.pseudovectorp =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_ALWAYS_INLINE,
+ comp.bool_type,
+ "PSEUDOVECTORP",
+ 2,
+ param,
+ 0);
+
+ DECL_BLOCK (entry_block, comp.pseudovectorp);
+ DECL_BLOCK (ret_false_b, comp.pseudovectorp);
+ DECL_BLOCK (call_pseudovector_typep_b, comp.pseudovectorp);
+
+ comp.block = entry_block;
+ comp.func = comp.pseudovectorp;
+
+ emit_cond_jump (emit_VECTORLIKEP (gcc_jit_param_as_rvalue (param[0])),
+ call_pseudovector_typep_b,
+ ret_false_b);
+
+ comp.block = ret_false_b;
+ gcc_jit_block_end_with_return (ret_false_b,
+ NULL,
+ gcc_jit_context_new_rvalue_from_int (
+ comp.ctxt,
+ comp.bool_type,
+ false));
+
+ gcc_jit_rvalue *args[] =
+ { gcc_jit_param_as_rvalue (param[0]),
+ gcc_jit_param_as_rvalue (param[1]) };
+ comp.block = call_pseudovector_typep_b;
+ /* FIXME use XUNTAG now that's available. */
+ gcc_jit_block_end_with_return (
+ call_pseudovector_typep_b,
+ NULL,
+ emit_call (intern_c_string ("helper_PSEUDOVECTOR_TYPEP_XUNTAG"),
+ comp.bool_type, 2, args, false));
+}
+
+static void
+define_CHECK_IMPURE (void)
+{
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ "obj"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.void_ptr_type,
+ "ptr") };
+ comp.check_impure =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_ALWAYS_INLINE,
+ comp.void_type,
+ "CHECK_IMPURE",
+ 2,
+ param,
+ 0);
+
+ DECL_BLOCK (entry_block, comp.check_impure);
+ DECL_BLOCK (err_block, comp.check_impure);
+ DECL_BLOCK (ok_block, comp.check_impure);
+
+ comp.block = entry_block;
+ comp.func = comp.check_impure;
+
+ emit_cond_jump (emit_PURE_P (gcc_jit_param_as_rvalue (param[0])), /* FIXME */
+ err_block,
+ ok_block);
+ gcc_jit_block_end_with_void_return (ok_block, NULL);
+
+ gcc_jit_rvalue *pure_write_error_arg =
+ gcc_jit_param_as_rvalue (param[0]);
+
+ comp.block = err_block;
+ gcc_jit_block_add_eval (comp.block,
+ NULL,
+ emit_call (intern_c_string ("pure_write_error"),
+ comp.void_type, 1,&pure_write_error_arg,
+ false));
+
+ gcc_jit_block_end_with_void_return (err_block, NULL);
+}
+
+/* Define a function to convert boolean into t or nil */
+
+static void
+define_bool_to_lisp_obj (void)
+{
+ /* x ? Qt : Qnil */
+ gcc_jit_param *param = gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.bool_type,
+ "x");
+ comp.bool_to_lisp_obj =
+ gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_ALWAYS_INLINE,
+ comp.lisp_obj_type,
+ "bool_to_lisp_obj",
+ 1,
+ ¶m,
+ 0);
+ DECL_BLOCK (entry_block, comp.bool_to_lisp_obj);
+ DECL_BLOCK (ret_t_block, comp.bool_to_lisp_obj);
+ DECL_BLOCK (ret_nil_block, comp.bool_to_lisp_obj);
+ comp.block = entry_block;
+ comp.func = comp.bool_to_lisp_obj;
+
+ emit_cond_jump (gcc_jit_param_as_rvalue (param),
+ ret_t_block,
+ ret_nil_block);
+
+ comp.block = ret_t_block;
+ gcc_jit_block_end_with_return (ret_t_block,
+ NULL,
+ emit_const_lisp_obj (Qt));
+
+ comp.block = ret_nil_block;
+ gcc_jit_block_end_with_return (ret_nil_block,
+ NULL,
+ emit_const_lisp_obj (Qnil));
+
+}
+
+/* Declare a function being compiled and add it to comp.exported_funcs_h. */
+
+static void
+declare_function (Lisp_Object func)
+{
+ gcc_jit_function *gcc_func;
+ char *c_name = SSDATA (CALL1I (comp-func-c-name, func));
+ Lisp_Object args = CALL1I (comp-func-args, func);
+ bool nargs = (CALL1I (comp-nargs-p, args));
+ USE_SAFE_ALLOCA;
+
+ if (!nargs)
+ {
+ EMACS_INT max_args = XFIXNUM (CALL1I (comp-args-max, args));
+ gcc_jit_type **type = SAFE_ALLOCA (max_args * sizeof (*type));
+ for (ptrdiff_t i = 0; i < max_args; i++)
+ type[i] = comp.lisp_obj_type;
+
+ gcc_jit_param **param = SAFE_ALLOCA (max_args *sizeof (*param));
+ for (int i = max_args - 1; i >= 0; i--)
+ param[i] = gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ type[i],
+ format_string ("par_%d", i));
+ gcc_func = gcc_jit_context_new_function (comp.ctxt, NULL,
+ GCC_JIT_FUNCTION_EXPORTED,
+ comp.lisp_obj_type,
+ c_name,
+ max_args,
+ param,
+ 0);
+ }
+ else
+ {
+ gcc_jit_param *param[] =
+ { gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.ptrdiff_type,
+ "nargs"),
+ gcc_jit_context_new_param (comp.ctxt,
+ NULL,
+ comp.lisp_obj_ptr_type,
+ "args") };
+ gcc_func =
+ gcc_jit_context_new_function (comp.ctxt,
+ NULL,
+ GCC_JIT_FUNCTION_EXPORTED,
+ comp.lisp_obj_type,
+ c_name, 2, param, 0);
+ }
+
+ Fputhash (CALL1I (comp-func-name, func),
+ make_mint_ptr (gcc_func),
+ comp.exported_funcs_h);
+
+ SAFE_FREE ();
+}
+
+static void
+compile_function (Lisp_Object func)
+{
+ USE_SAFE_ALLOCA;
+ EMACS_INT frame_size = XFIXNUM (CALL1I (comp-func-frame-size, func));
+
+ comp.func = xmint_pointer (Fgethash (CALL1I (comp-func-name, func),
+ comp.exported_funcs_h, Qnil));
+
+ gcc_jit_lvalue *frame_array =
+ gcc_jit_function_new_local (
+ comp.func,
+ NULL,
+ gcc_jit_context_new_array_type (comp.ctxt,
+ NULL,
+ comp.lisp_obj_type,
+ frame_size),
+ "local");
+ comp.frame = SAFE_ALLOCA (frame_size * sizeof (*comp.frame));
+ for (EMACS_INT i = 0; i < frame_size; ++i)
+ comp.frame[i] =
+ gcc_jit_context_new_array_access (
+ comp.ctxt,
+ NULL,
+ gcc_jit_lvalue_as_rvalue (frame_array),
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.int_type,
+ i));
+
+ /*
+ The floating frame is a copy of the normal frame that can be used to store
+ locals if the are not going to be used in a nargs call.
+ This has two advantages:
+ - Enable gcc for better reordering (frame array is clobbered every time is
+ passed as parameter being involved into an nargs function call).
+ - Allow gcc to trigger other optimizations that are prevented by memory
+ referencing.
+ */
+ if (SPEED >= 2)
+ {
+ comp.f_frame = SAFE_ALLOCA (frame_size * sizeof (*comp.f_frame));
+ for (ptrdiff_t i = 0; i < frame_size; ++i)
+ comp.f_frame[i] =
+ gcc_jit_function_new_local (comp.func,
+ NULL,
+ comp.lisp_obj_type,
+ format_string ("local%td", i));
+ }
+
+ comp.scratch = NULL;
+
+ comp.loc_handler = gcc_jit_function_new_local (comp.func,
+ NULL,
+ comp.handler_ptr_type,
+ "c");
+
+ comp.func_blocks_h = CALLN (Fmake_hash_table);
+
+ /* Pre-declare all basic blocks to gcc.
+ The "entry" block must be declared as first. */
+ declare_block (Qentry);
+ Lisp_Object blocks = CALL1I (comp-func-blocks, func);
+ Lisp_Object entry_block = Fgethash (Qentry, blocks, Qnil);
+ struct Lisp_Hash_Table *ht = XHASH_TABLE (blocks);
+ for (ptrdiff_t i = 0; i < ht->count; i++)
+ {
+ Lisp_Object block = HASH_VALUE (ht, i);
+ if (!EQ (block, entry_block))
+ declare_block (HASH_KEY (ht, i));
+ }
+
+ for (ptrdiff_t i = 0; i < ht->count; i++)
+ {
+ Lisp_Object block_name = HASH_KEY (ht, i);
+ Lisp_Object block = HASH_VALUE (ht, i);
+ Lisp_Object insns = CALL1I (comp-block-insns, block);
+ if (NILP (block) || NILP (insns))
+ xsignal1 (Qnative_ice,
+ build_string ("basic block is missing or empty"));
+
+ comp.block = retrive_block (block_name);
+ while (CONSP (insns))
+ {
+ Lisp_Object insn = XCAR (insns);
+ emit_limple_insn (insn);
+ insns = XCDR (insns);
+ }
+ }
+ const char *err = gcc_jit_context_get_first_error (comp.ctxt);
+ if (err)
+ xsignal3 (Qnative_ice,
+ build_string ("failing to compile function"),
+ CALL1I (comp-func-name, func),
+ build_string (err));
+
+ SAFE_FREE ();
+}
+
+\f
+/**********************************/
+/* Entry points exposed to lisp. */
+/**********************************/
+
+DEFUN ("comp--init-ctxt", Fcomp__init_ctxt, Scomp__init_ctxt,
+ 0, 0, 0,
+ doc: /* Initialize the native compiler context. Return t on success. */)
+ (void)
+{
+ if (comp.ctxt)
+ {
+ xsignal1 (Qnative_ice,
+ build_string ("compiler context already taken"));
+ return Qnil;
+ }
+
+ if (NILP (comp.emitter_dispatcher))
+ {
+ /* Move this into syms_of_comp the day will be dumpable. */
+ comp.emitter_dispatcher = CALLN (Fmake_hash_table);
+ register_emitter (Qset_internal, emit_set_internal);
+ register_emitter (Qhelper_unbind_n, emit_simple_limple_call_lisp_ret);
+ register_emitter (Qhelper_unwind_protect,
+ emit_simple_limple_call_void_ret);
+ register_emitter (Qrecord_unwind_current_buffer,
+ emit_simple_limple_call_lisp_ret);
+ register_emitter (Qrecord_unwind_protect_excursion,
+ emit_simple_limple_call_void_ret);
+ register_emitter (Qhelper_save_restriction,
+ emit_simple_limple_call_void_ret);
+ /* Inliners. */
+ register_emitter (Qadd1, emit_add1);
+ register_emitter (Qsub1, emit_sub1);
+ register_emitter (Qconsp, emit_consp);
+ register_emitter (Qcar, emit_car);
+ register_emitter (Qcdr, emit_cdr);
+ register_emitter (Qsetcar, emit_setcar);
+ register_emitter (Qsetcdr, emit_setcdr);
+ register_emitter (Qnegate, emit_negate);
+ register_emitter (Qnumberp, emit_numperp);
+ register_emitter (Qintegerp, emit_integerp);
+ }
+
+ comp.ctxt = gcc_jit_context_acquire ();
+
+ if (COMP_DEBUG)
+ {
+ gcc_jit_context_set_bool_option (comp.ctxt,
+ GCC_JIT_BOOL_OPTION_DEBUGINFO,
+ 1);
+ }
+ if (COMP_DEBUG > 1)
+ {
+ logfile = fopen ("libgccjit.log", "w");
+ gcc_jit_context_set_logfile (comp.ctxt,
+ logfile,
+ 0, 0);
+ gcc_jit_context_set_bool_option (comp.ctxt,
+ GCC_JIT_BOOL_OPTION_KEEP_INTERMEDIATES,
+ 1);
+ gcc_jit_context_set_bool_option (comp.ctxt,
+ GCC_JIT_BOOL_OPTION_DUMP_EVERYTHING,
+ 1);
+ }
+
+ comp.void_type = gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_VOID);
+ comp.void_ptr_type =
+ gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_VOID_PTR);
+ comp.bool_type = gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_BOOL);
+ comp.char_type = gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_CHAR);
+ comp.int_type = gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_INT);
+ comp.unsigned_type = gcc_jit_context_get_type (comp.ctxt,
+ GCC_JIT_TYPE_UNSIGNED_INT);
+ comp.long_type = gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_LONG);
+ comp.unsigned_long_type =
+ gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_UNSIGNED_LONG);
+ comp.long_long_type =
+ gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_LONG_LONG);
+ comp.unsigned_long_long_type =
+ gcc_jit_context_get_type (comp.ctxt, GCC_JIT_TYPE_UNSIGNED_LONG_LONG);
+ comp.char_ptr_type = gcc_jit_type_get_pointer (comp.char_type);
+
+#if EMACS_INT_MAX <= LONG_MAX
+ /* 32-bit builds without wide ints, 64-bit builds on Posix hosts. */
+ comp.lisp_obj_as_ptr = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.void_ptr_type,
+ "obj");
+#else
+ /* 64-bit builds on MS-Windows, 32-bit builds with wide ints. */
+ comp.lisp_obj_as_ptr = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.long_long_type,
+ "obj");
+#endif
+
+ comp.emacs_int_type = gcc_jit_context_get_int_type (comp.ctxt,
+ sizeof (EMACS_INT),
+ true);
+
+ comp.lisp_obj_as_num = gcc_jit_context_new_field (comp.ctxt,
+ NULL,
+ comp.emacs_int_type,
+ "num");
+
+ gcc_jit_field *lisp_obj_fields[] = { comp.lisp_obj_as_ptr,
+ comp.lisp_obj_as_num };
+ comp.lisp_obj_type = gcc_jit_context_new_union_type (comp.ctxt,
+ NULL,
+ "comp_Lisp_Object",
+ sizeof (lisp_obj_fields)
+ / sizeof (*lisp_obj_fields),
+ lisp_obj_fields);
+ comp.lisp_obj_ptr_type = gcc_jit_type_get_pointer (comp.lisp_obj_type);
+
+ comp.most_positive_fixnum =
+ gcc_jit_context_new_rvalue_from_long (comp.ctxt,
+ comp.emacs_int_type,
+ MOST_POSITIVE_FIXNUM);
+ comp.most_negative_fixnum =
+ gcc_jit_context_new_rvalue_from_long (comp.ctxt,
+ comp.emacs_int_type,
+ MOST_NEGATIVE_FIXNUM);
+ comp.one =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ 1);
+ comp.inttypebits =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ INTTYPEBITS);
+
+ comp.lisp_int0 =
+ gcc_jit_context_new_rvalue_from_int (comp.ctxt,
+ comp.emacs_int_type,
+ Lisp_Int0);
+
+ comp.ptrdiff_type = gcc_jit_context_get_int_type (comp.ctxt,
+ sizeof (void *),
+ true);
+
+ comp.uintptr_type = gcc_jit_context_get_int_type (comp.ctxt,
+ sizeof (void *),
+ false);
+
+ comp.exported_funcs_h = CALLN (Fmake_hash_table);
+ /*
+ Always reinitialize this cause old function definitions are garbage collected
+ by libgccjit when the ctxt is released.
+ */
+ comp.imported_funcs_h = CALLN (Fmake_hash_table);
+
+ /* Define data structures. */
+
+ define_lisp_cons ();
+ define_jmp_buf ();
+ define_handler_struct ();
+ define_thread_state_struct ();
+ define_cast_union ();
+
+ return Qt;
+}
+
+DEFUN ("comp--release-ctxt", Fcomp__release_ctxt, Scomp__release_ctxt,
+ 0, 0, 0,
+ doc: /* Release the native compiler context. */)
+ (void)
+{
+ if (comp.ctxt)
+ gcc_jit_context_release (comp.ctxt);
+
+ if (logfile)
+ fclose (logfile);
+ comp.ctxt = NULL;
+
+ return Qt;
+}
+
+DEFUN ("comp--compile-ctxt-to-file", Fcomp__compile_ctxt_to_file,
+ Scomp__compile_ctxt_to_file,
+ 1, 1, 0,
+ doc: /* Compile as native code the current context to file. */)
+ (Lisp_Object ctxtname)
+{
+ CHECK_STRING (ctxtname);
+
+ Frequire (Qadvice, Qnil, Qnil);
+
+ gcc_jit_context_set_int_option (comp.ctxt,
+ GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL,
+ SPEED);
+ /* Gcc doesn't like being interrupted at all. */
+ block_input ();
+ sigset_t oldset;
+ sigset_t blocked;
+ sigemptyset (&blocked);
+ sigaddset (&blocked, SIGALRM);
+ sigaddset (&blocked, SIGINT);
+ sigaddset (&blocked, SIGIO);
+ pthread_sigmask (SIG_BLOCK, &blocked, &oldset);
+
+ emit_ctxt_code ();
+
+ /* Define inline functions. */
+ define_CAR_CDR ();
+ define_PSEUDOVECTORP ();
+ define_CHECK_TYPE ();
+ define_CHECK_IMPURE ();
+ define_bool_to_lisp_obj ();
+ define_setcar_setcdr ();
+ define_add1_sub1 ();
+ define_negate ();
+
+ struct Lisp_Hash_Table *func_h
+ = XHASH_TABLE (CALL1I (comp-ctxt-funcs-h, Vcomp_ctxt));
+ for (ptrdiff_t i = 0; i < func_h->count; i++)
+ declare_function (HASH_VALUE (func_h, i));
+ /* Compile all functions. Can't be done before because the
+ relocation structs has to be already defined. */
+ for (ptrdiff_t i = 0; i < func_h->count; i++)
+ compile_function (HASH_VALUE (func_h, i));
+
+ if (COMP_DEBUG)
+ gcc_jit_context_dump_to_file (comp.ctxt,
+ format_string ("%s.c", SSDATA (ctxtname)),
+ 1);
+ if (COMP_DEBUG > 2)
+ gcc_jit_context_dump_reproducer_to_file (comp.ctxt, "comp_reproducer.c");
+
+ AUTO_STRING (dot_so, NATIVE_ELISP_SUFFIX);
+
+ Lisp_Object out_file = CALLN (Fconcat, ctxtname, dot_so);
+
+ /* Remove the old eln before creating the new one to get a new inode and
+ prevent crashes in case the old one is currently loaded. */
+ if (!NILP (Ffile_exists_p (out_file)))
+ Fdelete_file (out_file, Qnil);
+
+ gcc_jit_context_compile_to_file (comp.ctxt,
+ GCC_JIT_OUTPUT_KIND_DYNAMIC_LIBRARY,
+ SSDATA (out_file));
+
+ pthread_sigmask (SIG_SETMASK, &oldset, 0);
+ unblock_input ();
+
+ return out_file;
+}
+
+\f
+/******************************************************************************/
+/* Helper functions called from the run-time. */
+/* These can't be statics till shared mechanism is used to solve relocations. */
+/* Note: this are all potentially definable directly to gcc and are here just */
+/* for laziness. Change this if a performance impact is measured. */
+/******************************************************************************/
+
+Lisp_Object
+helper_save_window_excursion (Lisp_Object v1)
+{
+ ptrdiff_t count1 = SPECPDL_INDEX ();
+ record_unwind_protect (restore_window_configuration,
+ Fcurrent_window_configuration (Qnil));
+ v1 = Fprogn (v1);
+ unbind_to (count1, v1);
+ return v1;
+}
+
+void
+helper_unwind_protect (Lisp_Object handler)
+{
+ /* Support for a function here is new in 24.4. */
+ record_unwind_protect (FUNCTIONP (handler) ? bcall0 : prog_ignore,
+ handler);
+}
+
+Lisp_Object
+helper_temp_output_buffer_setup (Lisp_Object x)
+{
+ CHECK_STRING (x);
+ temp_output_buffer_setup (SSDATA (x));
+ return Vstandard_output;
+}
+
+Lisp_Object
+helper_unbind_n (Lisp_Object n)
+{
+ return unbind_to (SPECPDL_INDEX () - XFIXNUM (n), Qnil);
+}
+
+void
+helper_save_restriction (void)
+{
+ record_unwind_protect (save_restriction_restore,
+ save_restriction_save ());
+}
+
+bool
+helper_PSEUDOVECTOR_TYPEP_XUNTAG (Lisp_Object a, enum pvec_type code)
+{
+ return PSEUDOVECTOR_TYPEP (XUNTAG (a, Lisp_Vectorlike,
+ union vectorlike_header),
+ code);
+}
+
+\f
+/**************************************/
+/* Functions used to load eln files. */
+/**************************************/
+
+static Lisp_Object Vnative_elisp_refs_hash;
+static Lisp_Object load_handle_stack;
+
+static void
+prevent_gc (Lisp_Object obj)
+{
+ Fputhash (obj, Qt, Vnative_elisp_refs_hash);
+}
+
+typedef char *(*comp_lit_str_func) (void);
+
+/* Deserialize read and return static object. */
+static Lisp_Object
+load_static_obj (dynlib_handle_ptr handle, const char *name)
+{
+ static_obj_t *(*f)(void) = dynlib_sym (handle, name);
+ eassert (f);
+ static_obj_t *res = f ();
+ return Fread (make_string (res->data, res->len));
+}
+
+static void
+load_comp_unit (dynlib_handle_ptr handle, Lisp_Object file)
+{
+ struct thread_state ***current_thread_reloc =
+ dynlib_sym (handle, CURRENT_THREAD_RELOC_SYM);
+ EMACS_INT ***pure_reloc = dynlib_sym (handle, PURE_RELOC_SYM);
+ Lisp_Object *data_relocs = dynlib_sym (handle, DATA_RELOC_SYM);
+ Lisp_Object (**f_relocs)(void) = dynlib_sym (handle, IMPORTED_FUNC_RELOC_SYM);
+ void (*top_level_run)(void) = dynlib_sym (handle, "top_level_run");
+
+ if (!(current_thread_reloc
+ && pure_reloc
+ && data_relocs
+ && f_relocs
+ && top_level_run))
+ xsignal1 (Qnative_lisp_file_inconsistent, file);
+
+ *current_thread_reloc = ¤t_thread;
+ *pure_reloc = (EMACS_INT **)&pure;
+
+ /* Imported data. */
+ Lisp_Object d_vec = load_static_obj (handle, TEXT_DATA_RELOC_SYM);
+ EMACS_INT d_vec_len = XFIXNUM (Flength (d_vec));
+
+ for (EMACS_INT i = 0; i < d_vec_len; i++)
+ {
+ data_relocs[i] = AREF (d_vec, i);
+ prevent_gc (data_relocs[i]);
+ }
+
+ /* Imported functions. */
+ Lisp_Object f_vec =
+ load_static_obj (handle, TEXT_IMPORTED_FUNC_RELOC_SYM);
+ EMACS_INT f_vec_len = XFIXNUM (Flength (f_vec));
+ for (EMACS_INT i = 0; i < f_vec_len; i++)
+ {
+ Lisp_Object f_sym = AREF (f_vec, i);
+ char *f_str = SSDATA (SYMBOL_NAME (f_sym));
+ Lisp_Object subr = Fsymbol_function (f_sym);
+ if (!NILP (subr))
+ {
+ subr = symbol_subr (f_sym);
+ if (NILP (subr))
+ /* FIXME: This is not robust in case of primitive
+ redefinition. */
+ xsignal2 (Qnative_lisp_wrong_reloc, f_sym, file);
+
+ f_relocs[i] = XSUBR (subr)->function.a0;
+ }
+ else if (!strcmp (f_str, "wrong_type_argument"))
+ f_relocs[i] = (void *) wrong_type_argument;
+ else if (!strcmp (f_str, "helper_PSEUDOVECTOR_TYPEP_XUNTAG"))
+ f_relocs[i] = (void *) helper_PSEUDOVECTOR_TYPEP_XUNTAG;
+ else if (!strcmp (f_str, "pure_write_error"))
+ f_relocs[i] = (void *) pure_write_error;
+ else if (!strcmp (f_str, "push_handler"))
+ f_relocs[i] = (void *) push_handler;
+ else if (!strcmp (f_str, SETJMP_NAME))
+ f_relocs[i] = (void *) SETJMP;
+ else if (!strcmp (f_str, "record_unwind_protect_excursion"))
+ f_relocs[i] = (void *) record_unwind_protect_excursion;
+ else if (!strcmp (f_str, "helper_unbind_n"))
+ f_relocs[i] = (void *) helper_unbind_n;
+ else if (!strcmp (f_str, "helper_save_restriction"))
+ f_relocs[i] = (void *) helper_save_restriction;
+ else if (!strcmp (f_str, "record_unwind_current_buffer"))
+ f_relocs[i] = (void *) record_unwind_current_buffer;
+ else if (!strcmp (f_str, "set_internal"))
+ f_relocs[i] = (void *) set_internal;
+ else if (!strcmp (f_str, "helper_unwind_protect"))
+ f_relocs[i] = (void *) helper_unwind_protect;
+ else if (!strcmp (f_str, "specbind"))
+ f_relocs[i] = (void *) specbind;
+ else
+ xsignal2 (Qnative_lisp_wrong_reloc, f_sym, file);
+ }
+
+ /* Executing this will perform all the expected environment modification. */
+ top_level_run ();
+
+ return;
+}
+
+DEFUN ("comp--register-subr", Fcomp__register_subr,
+ Scomp__register_subr,
+ 5, 5, 0,
+ doc: /* This gets called by top_level_run during load phase to register
+ each exported subr. */)
+ (Lisp_Object name, Lisp_Object minarg, Lisp_Object maxarg,
+ Lisp_Object c_name, Lisp_Object doc)
+{
+ dynlib_handle_ptr handle = xmint_pointer (XCAR (load_handle_stack));
+ if (!handle)
+ xsignal0 (Qwrong_register_subr_call);
+
+ void *func = dynlib_sym (handle, SSDATA (c_name));
+ eassert (func);
+
+ union Aligned_Lisp_Subr *x = xmalloc (sizeof (union Aligned_Lisp_Subr));
+ x->s.header.size = PVEC_SUBR << PSEUDOVECTOR_AREA_BITS;
+ x->s.function.a0 = func;
+ x->s.min_args = XFIXNUM (minarg);
+ x->s.max_args = FIXNUMP (maxarg) ? XFIXNUM (maxarg) : MANY;
+ x->s.symbol_name = SSDATA (Fsymbol_name (name));
+ x->s.intspec = NULL;
+ x->s.doc = 0; /* FIXME */
+ x->s.native_elisp = true;
+ defsubr (x);
+
+ return Qnil;
+}
+
+/* Load related routines. */
+DEFUN ("native-elisp-load", Fnative_elisp_load, Snative_elisp_load, 1, 1, 0,
+ doc: /* Load native elisp code FILE. */)
+ (Lisp_Object file)
+{
+ CHECK_STRING (file);
+
+ Frequire (Qadvice, Qnil, Qnil);
+
+ dynlib_handle_ptr handle = dynlib_open (SSDATA (file));
+ load_handle_stack = Fcons (make_mint_ptr (handle), load_handle_stack);
+ if (!handle)
+ xsignal2 (Qnative_lisp_load_failed, file, build_string (dynlib_error ()));
+
+ load_comp_unit (handle, file);
+
+ load_handle_stack = XCDR (load_handle_stack);
+
+ return Qt;
+}
+
+\f
+void
+syms_of_comp (void)
+{
+ /* Compiler control customizes. */
+ DEFSYM (Qcomp_speed, "comp-speed");
+ DEFSYM (Qcomp_debug, "comp-debug");
+
+ /* Limple instruction set. */
+ DEFSYM (Qcomment, "comment");
+ DEFSYM (Qjump, "jump");
+ DEFSYM (Qcall, "call");
+ DEFSYM (Qcallref, "callref");
+ DEFSYM (Qdirect_call, "direct-call");
+ DEFSYM (Qdirect_callref, "direct-callref");
+ DEFSYM (Qsetimm, "setimm");
+ DEFSYM (Qreturn, "return");
+ DEFSYM (Qcomp_mvar, "comp-mvar");
+ DEFSYM (Qcond_jump, "cond-jump");
+ DEFSYM (Qphi, "phi");
+ /* Ops in use for prologue emission. */
+ DEFSYM (Qset_par_to_local, "set-par-to-local");
+ DEFSYM (Qset_args_to_local, "set-args-to-local");
+ DEFSYM (Qset_rest_args_to_local, "set-rest-args-to-local");
+ DEFSYM (Qinc_args, "inc-args");
+ DEFSYM (Qcond_jump_narg_leq, "cond-jump-narg-leq");
+ /* Others. */
+ DEFSYM (Qpush_handler, "push-handler");
+ DEFSYM (Qpop_handler, "pop-handler");
+ DEFSYM (Qfetch_handler, "fetch-handler");
+ DEFSYM (Qcondition_case, "condition-case");
+ /* call operands. */
+ DEFSYM (Qcatcher, "catcher");
+ DEFSYM (Qentry, "entry");
+ DEFSYM (Qset_internal, "set_internal");
+ DEFSYM (Qrecord_unwind_current_buffer, "record_unwind_current_buffer");
+ DEFSYM (Qrecord_unwind_protect_excursion, "record_unwind_protect_excursion");
+ DEFSYM (Qhelper_unbind_n, "helper_unbind_n");
+ DEFSYM (Qhelper_unwind_protect, "helper_unwind_protect");
+ DEFSYM (Qhelper_save_restriction, "helper_save_restriction");
+ /* Inliners. */
+ DEFSYM (Qadd1, "1+");
+ DEFSYM (Qsub1, "1-");
+ DEFSYM (Qconsp, "consp");
+ DEFSYM (Qcar, "car");
+ DEFSYM (Qcdr, "cdr");
+ DEFSYM (Qsetcar, "setcar");
+ DEFSYM (Qsetcdr, "setcdr");
+ DEFSYM (Qnegate, "negate");
+ DEFSYM (Qnumberp, "numberp");
+ DEFSYM (Qintegerp, "integerp");
+
+ /* Others. */
+ DEFSYM (Qfixnum, "fixnum");
+ DEFSYM (Qadvice, "advice");
+
+ /* To be signaled. */
+
+ /* By the compiler. */
+ DEFSYM (Qnative_compiler_error, "native-compiler-error");
+ Fput (Qnative_compiler_error, Qerror_conditions,
+ pure_list (Qnative_compiler_error, Qerror));
+ Fput (Qnative_compiler_error, Qerror_message,
+ build_pure_c_string ("Native compiler error"));
+
+ DEFSYM (Qnative_ice, "native-ice");
+ Fput (Qnative_ice, Qerror_conditions,
+ pure_list (Qnative_ice, Qnative_compiler_error, Qerror));
+ Fput (Qnative_ice, Qerror_message,
+ build_pure_c_string ("Internal native compiler error"));
+
+ /* By the load machinery. */
+ DEFSYM (Qnative_lisp_load_failed, "native-lisp-load-failed");
+ Fput (Qnative_lisp_load_failed, Qerror_conditions,
+ pure_list (Qnative_lisp_load_failed, Qerror));
+ Fput (Qnative_lisp_load_failed, Qerror_message,
+ build_pure_c_string ("Native elisp load failed"));
+
+ DEFSYM (Qnative_lisp_wrong_reloc, "native-lisp-wrong-reloc");
+ Fput (Qnative_lisp_wrong_reloc, Qerror_conditions,
+ pure_list (Qnative_lisp_wrong_reloc, Qnative_lisp_load_failed, Qerror));
+ Fput (Qnative_lisp_wrong_reloc, Qerror_message,
+ build_pure_c_string ("Primitive redefined or wrong relocation"));
+
+ DEFSYM (Qwrong_register_subr_call, "wrong-register-subr-call");
+ Fput (Qwrong_register_subr_call, Qerror_conditions,
+ pure_list (Qwrong_register_subr_call, Qnative_lisp_load_failed, Qerror));
+ Fput (Qwrong_register_subr_call, Qerror_message,
+ build_pure_c_string ("comp--register-subr can only be called during "
+ "native lisp load phase."));
+
+ DEFSYM (Qnative_lisp_file_inconsistent, "native-lisp-file-inconsistent");
+ Fput (Qnative_lisp_file_inconsistent, Qerror_conditions,
+ pure_list (Qnative_lisp_file_inconsistent, Qnative_lisp_load_failed, Qerror));
+ Fput (Qnative_lisp_file_inconsistent, Qerror_message,
+ build_pure_c_string ("inconsistent eln file"));
+
+ defsubr (&Scomp__init_ctxt);
+ defsubr (&Scomp__release_ctxt);
+ defsubr (&Scomp__compile_ctxt_to_file);
+ defsubr (&Scomp__register_subr);
+ defsubr (&Snative_elisp_load);
+
+ staticpro (&comp.exported_funcs_h);
+ comp.exported_funcs_h = Qnil;
+ staticpro (&comp.imported_funcs_h);
+ comp.imported_funcs_h = Qnil;
+ staticpro (&comp.func_blocks_h);
+ staticpro (&comp.emitter_dispatcher);
+ comp.emitter_dispatcher = Qnil;
+
+ DEFVAR_LISP ("comp-ctxt", Vcomp_ctxt,
+ doc: /* The compiler context. */);
+ Vcomp_ctxt = Qnil;
+
+ /* Load mechanism. */
+ staticpro (&Vnative_elisp_refs_hash);
+ Vnative_elisp_refs_hash
+ = make_hash_table (hashtest_eq, DEFAULT_HASH_SIZE,
+ DEFAULT_REHASH_SIZE, DEFAULT_REHASH_THRESHOLD,
+ Qnil, false);
+ staticpro (&load_handle_stack);
+ load_handle_stack = Qnil;
+}
+
+#endif /* HAVE_NATIVE_COMP */
diff --git a/src/data.c b/src/data.c
index 26e8611..f379711 100644
--- a/src/data.c
+++ b/src/data.c
@@ -864,6 +864,16 @@ DEFUN ("subr-name", Fsubr_name, Ssubr_name, 1, 1, 0,
return build_string (name);
}
+#ifdef HAVE_NATIVE_COMP
+DEFUN ("subr-native-elisp-p", Fsubr_native_elisp_p, Ssubr_native_elisp_p, 1, 1, 0,
+ doc: /* Return t if the object is native compiled lisp function,
+nil otherwise. */)
+ (Lisp_Object object)
+{
+ return (SUBRP (object) && XSUBR (object)->native_elisp) ? Qt : Qnil;
+}
+#endif
+
DEFUN ("interactive-form", Finteractive_form, Sinteractive_form, 1, 1, 0,
doc: /* Return the interactive form of CMD or nil if none.
If CMD is not a command, the return value is nil.
@@ -3998,6 +4008,9 @@ #define PUT_ERROR(sym, tail, msg) \
defsubr (&Sbyteorder);
defsubr (&Ssubr_arity);
defsubr (&Ssubr_name);
+#ifdef HAVE_NATIVE_COMP
+ defsubr (&Ssubr_native_elisp_p);
+#endif
#ifdef HAVE_MODULES
defsubr (&Suser_ptrp);
#endif
diff --git a/src/emacs.c b/src/emacs.c
index 8a6e34d..071d0cc 100644
--- a/src/emacs.c
+++ b/src/emacs.c
@@ -1598,6 +1598,11 @@ main (int argc, char **argv)
init_json ();
#endif
+#ifdef HAVE_NATIVE_COMP
+ if (!initialized)
+ syms_of_comp ();
+#endif
+
no_loadup
= argmatch (argv, argc, "-nl", "--no-loadup", 6, NULL, &skip_args);
diff --git a/src/eval.c b/src/eval.c
index 4bc96f9..ca96424 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -219,8 +219,14 @@ backtrace_thread_next (struct thread_state *tstate, union specbinding *pdl)
init_eval_once (void)
{
/* Don't forget to update docs (lispref node "Local Variables"). */
+#ifndef HAVE_NATIVE_COMP
max_specpdl_size = 1600; /* 1500 is not enough for cl-generic.el. */
max_lisp_eval_depth = 800;
+#else
+ /* Original values increased for comp.el. */
+ max_specpdl_size = 2100;
+ max_lisp_eval_depth = 1400;
+#endif
Vrun_hooks = Qnil;
pdumper_do_now_and_after_load (init_eval_once_for_pdumper);
}
diff --git a/src/lisp.h b/src/lisp.h
index e0ae2c4..aa891bb 100644
--- a/src/lisp.h
+++ b/src/lisp.h
@@ -2088,6 +2088,9 @@ CHAR_TABLE_SET (Lisp_Object ct, int idx, Lisp_Object val)
const char *symbol_name;
const char *intspec;
EMACS_INT doc;
+#ifdef HAVE_NATIVE_COMP
+ bool native_elisp;
+#endif
} GCALIGNED_STRUCT;
union Aligned_Lisp_Subr
{
@@ -4741,6 +4744,10 @@ maybe_disable_address_randomization (int argc, char **argv)
extern void malloc_probe (size_t);
extern void syms_of_profiler (void);
+/* Defined in comp.c. */
+#ifdef HAVE_NATIVE_COMP
+extern void syms_of_comp (void);
+#endif /* HAVE_NATIVE_COMP */
#ifdef DOS_NT
/* Defined in msdos.c, w32.c. */
diff --git a/src/lread.c b/src/lread.c
index 4ea6202..92f90b2 100644
--- a/src/lread.c
+++ b/src/lread.c
@@ -1273,6 +1273,11 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
bool is_module = false;
#endif
+#ifdef HAVE_NATIVE_COMP
+ bool is_native_elisp = suffix_p (found, NATIVE_ELISP_SUFFIX);
+#else
+ bool is_native_elisp = false;
+#endif
/* Check if we're stuck in a recursive load cycle.
2000-09-21: It's not possible to just check for the file loaded
@@ -1371,7 +1376,7 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
} /* !load_prefer_newer */
}
}
- else if (!is_module)
+ else if (!is_module && !is_native_elisp)
{
/* We are loading a source file (*.el). */
if (!NILP (Vload_source_file_function))
@@ -1398,7 +1403,7 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
stream = NULL;
errno = EINVAL;
}
- else if (!is_module)
+ else if (!is_module && !is_native_elisp)
{
#ifdef WINDOWSNT
emacs_close (fd);
@@ -1414,7 +1419,7 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
might be accessed by the unbind_to call below. */
struct infile input;
- if (is_module)
+ if (is_module || is_native_elisp)
{
/* `module-load' uses the file name, so we can close the stream
now. */
@@ -1444,6 +1449,8 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
file, 1);
else if (is_module)
message_with_string ("Loading %s (module)...", file, 1);
+ else if (is_native_elisp)
+ message_with_string ("Loading %s (native compiled elisp)...", file, 1);
else if (!compiled)
message_with_string ("Loading %s (source)...", file, 1);
else if (newer)
@@ -1469,6 +1476,18 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
emacs_abort ();
#endif
}
+ else if (is_native_elisp)
+ {
+#ifdef HAVE_NATIVE_COMP
+ specbind (Qcurrent_load_list, Qnil);
+ LOADHIST_ATTACH (found);
+ Fnative_elisp_load (found);
+ build_load_history (found, true);
+#else
+ /* This cannot happen. */
+ emacs_abort ();
+#endif
+ }
else
{
if (lisp_file_lexically_bound_p (Qget_file_char))
@@ -1507,6 +1526,8 @@ DEFUN ("load", Fload, Sload, 1, 5, 0,
file, 1);
else if (is_module)
message_with_string ("Loading %s (module)...done", file, 1);
+ else if (is_native_elisp)
+ message_with_string ("Loading %s (native compiled elisp)...done", file, 1);
else if (!compiled)
message_with_string ("Loading %s (source)...done", file, 1);
else if (newer)
@@ -4862,14 +4883,15 @@ syms_of_lread (void)
This list should not include the empty string.
`load' and related functions try to append these suffixes, in order,
to the specified file name if a suffix is allowed or required. */);
-#ifdef HAVE_MODULES
- Vload_suffixes = list3 (build_pure_c_string (".elc"),
- build_pure_c_string (".el"),
- build_pure_c_string (MODULES_SUFFIX));
-#else
Vload_suffixes = list2 (build_pure_c_string (".elc"),
build_pure_c_string (".el"));
+#ifdef HAVE_MODULES
+ Vload_suffixes = Fcons (build_pure_c_string (MODULES_SUFFIX), Vload_suffixes);
#endif
+#ifdef HAVE_NATIVE_COMP
+ Vload_suffixes = Fcons (build_pure_c_string (NATIVE_ELISP_SUFFIX), Vload_suffixes);
+#endif
+
DEFVAR_LISP ("module-file-suffix", Vmodule_file_suffix,
doc: /* Suffix of loadable module file, or nil if modules are not supported. */);
#ifdef HAVE_MODULES
diff --git a/src/pdumper.c b/src/pdumper.c
index 74f198c..d998f5d 100644
--- a/src/pdumper.c
+++ b/src/pdumper.c
@@ -2922,7 +2922,10 @@ dump_bool_vector (struct dump_context *ctx, const struct Lisp_Vector *v)
static dump_off
dump_subr (struct dump_context *ctx, const struct Lisp_Subr *subr)
{
-#if CHECK_STRUCTS && !defined (HASH_Lisp_Subr_594AB72B54)
+#if CHECK_STRUCTS && ((defined (HAVE_NATIVE_COMP) \
+ && !defined (HASH_Lisp_Subr_D4F15794AF)) \
+ || (!defined (HAVE_NATIVE_COMP) \
+ && !defined (HASH_Lisp_Subr_594AB72B54)))
# error "Lisp_Subr changed. See CHECK_STRUCTS comment in config.h."
#endif
struct Lisp_Subr out;
@@ -2934,6 +2937,9 @@ dump_subr (struct dump_context *ctx, const struct Lisp_Subr *subr)
dump_field_emacs_ptr (ctx, &out, subr, &subr->symbol_name);
dump_field_emacs_ptr (ctx, &out, subr, &subr->intspec);
DUMP_FIELD_COPY (&out, subr, doc);
+#ifdef HAVE_NATIVE_COMP
+ DUMP_FIELD_COPY (&out, subr, native_elisp);
+#endif
return dump_object_finish (ctx, &out, sizeof (out));
}
diff --git a/test/src/comp-test-funcs.el b/test/src/comp-test-funcs.el
new file mode 100644
index 0000000..20d15ac
--- /dev/null
+++ b/test/src/comp-test-funcs.el
@@ -0,0 +1,393 @@
+;;; comp-test-funcs.el --- compilation unit tested by comp-tests.el -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; Author: Andrea Corallo <akrl@sdf.org>
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;;; Code:
+
+(defvar comp-tests-var1 3)
+
+(defun comp-tests-varref-f ()
+ comp-tests-var1)
+
+(defun comp-tests-list-f ()
+ (list 1 2 3))
+(defun comp-tests-list2-f (a b c)
+ (list a b c))
+(defun comp-tests-car-f (x)
+ ;; Bcar
+ (car x))
+(defun comp-tests-cdr-f (x)
+ ;; Bcdr
+ (cdr x))
+(defun comp-tests-car-safe-f (x)
+ ;; Bcar_safe
+ (car-safe x))
+(defun comp-tests-cdr-safe-f (x)
+ ;; Bcdr_safe
+ (cdr-safe x))
+
+(defun comp-tests-cons-car-f ()
+ (car (cons 1 2)))
+(defun comp-tests-cons-cdr-f (x)
+ (cdr (cons 'foo x)))
+
+(defun comp-tests-varset0-f ()
+ (setq comp-tests-var1 55))
+(defun comp-tests-varset1-f ()
+ (setq comp-tests-var1 66)
+ 4)
+
+(defun comp-tests-length-f ()
+ (length '(1 2 3)))
+
+(defun comp-tests-aref-aset-f ()
+ (let ((vec [1 2 3]))
+ (aset vec 2 100)
+ (aref vec 2)))
+
+(defvar comp-tests-var2 3)
+(defun comp-tests-symbol-value-f ()
+ (symbol-value 'comp-tests-var2))
+
+(defun comp-tests-concat-f (x)
+ (concat "a" "b" "c" "d"
+ (concat "a" "b" "c" (concat "a" "b" (concat "foo" x)))))
+
+(defun comp-tests-ffuncall-callee-f (x y z)
+ (list x y z))
+
+(defun comp-tests-ffuncall-callee-optional-f (a b &optional c d)
+ (list a b c d))
+
+(defun comp-tests-ffuncall-callee-rest-f (a b &rest c)
+ (list a b c))
+
+(defun comp-tests-ffuncall-callee-more8-f (p1 p2 p3 p4 p5 p6 p7 p8 p9 p10)
+ ;; More then 8 args.
+ (list p1 p2 p3 p4 p5 p6 p7 p8 p9 p10))
+
+(defun comp-tests-ffuncall-callee-more8-rest-f (p1 p2 p3 p4 p5 p6 p7 p8 p9 &rest p10)
+ ;; More then 8 args.
+ (list p1 p2 p3 p4 p5 p6 p7 p8 p9 p10))
+
+(defun comp-tests-ffuncall-native-f ()
+ "Call a primitive with no dedicate op."
+ (make-vector 1 nil))
+
+(defun comp-tests-ffuncall-native-rest-f ()
+ "Call a primitive with no dedicate op with &rest."
+ (vector 1 2 3))
+
+(defun comp-tests-ffuncall-apply-many-f (x)
+ (apply #'list x))
+
+(defun comp-tests-ffuncall-lambda-f (x)
+ (let ((fun (lambda (x)
+ (1+ x))))
+ (funcall fun x)))
+
+(defun comp-tests-jump-table-1-f (x)
+ (pcase x
+ ('x 'a)
+ ('y 'b)
+ (_ 'c)))
+
+(defun comp-tests-jump-table-2-f (x)
+ (pcase x
+ ("aaa" 'a)
+ ("bbb" 'b)))
+
+(defun comp-tests-conditionals-1-f (x)
+ ;; Generate goto-if-nil
+ (if x 1 2))
+(defun comp-tests-conditionals-2-f (x)
+ ;; Generate goto-if-nil-else-pop
+ (when x
+ 1340))
+
+(defun comp-tests-fixnum-1-minus-f (x)
+ ;; Bsub1
+ (1- x))
+(defun comp-tests-fixnum-1-plus-f (x)
+ ;; Badd1
+ (1+ x))
+(defun comp-tests-fixnum-minus-f (x)
+ ;; Bnegate
+ (- x))
+
+(defun comp-tests-eqlsign-f (x y)
+ ;; Beqlsign
+ (= x y))
+(defun comp-tests-gtr-f (x y)
+ ;; Bgtr
+ (> x y))
+(defun comp-tests-lss-f (x y)
+ ;; Blss
+ (< x y))
+(defun comp-tests-les-f (x y)
+ ;; Bleq
+ (<= x y))
+(defun comp-tests-geq-f (x y)
+ ;; Bgeq
+ (>= x y))
+
+(defun comp-tests-setcar-f (x y)
+ (setcar x y)
+ x)
+(defun comp-tests-setcdr-f (x y)
+ (setcdr x y)
+ x)
+
+(defun comp-bubble-sort-f (list)
+ (let ((i (length list)))
+ (while (> i 1)
+ (let ((b list))
+ (while (cdr b)
+ (when (< (cadr b) (car b))
+ (setcar b (prog1 (cadr b)
+ (setcdr b (cons (car b) (cddr b))))))
+ (setq b (cdr b))))
+ (setq i (1- i)))
+ list))
+
+(defun comp-tests-consp-f (x)
+ ;; Bconsp
+ (consp x))
+(defun comp-tests-setcar2-f (x)
+ ;; Bsetcar
+ (setcar x 3))
+
+(defun comp-tests-integerp-f (x)
+ ;; Bintegerp
+ (integerp x))
+(defun comp-tests-numberp-f (x)
+ ;; Bnumberp
+ (numberp x))
+
+(defun comp-tests-discardn-f (x)
+ ;; BdiscardN
+ (1+ (let ((a 1)
+ (_b)
+ (_c))
+ a)))
+(defun comp-tests-insertn-f (a b c d)
+ ;; Binsert
+ (insert a b c d))
+
+(defun comp-tests-err-arith-f ()
+ (/ 1 0))
+(defun comp-tests-err-foo-f ()
+ (error "foo"))
+
+(defun comp-tests-condition-case-0-f ()
+ ;; Bpushhandler Bpophandler
+ (condition-case
+ err
+ (comp-tests-err-arith-f)
+ (arith-error (concat "arith-error "
+ (error-message-string err)
+ " catched"))
+ (error (concat "error "
+ (error-message-string err)
+ " catched"))))
+(defun comp-tests-condition-case-1-f ()
+ ;; Bpushhandler Bpophandler
+ (condition-case
+ err
+ (comp-tests-err-foo-f)
+ (arith-error (concat "arith-error "
+ (error-message-string err)
+ " catched"))
+ (error (concat "error "
+ (error-message-string err)
+ " catched"))))
+(defun comp-tests-catch-f (f)
+ (catch 'foo
+ (funcall f)))
+(defun comp-tests-throw-f (x)
+ (throw 'foo x))
+
+(defun comp-tests-buff0-f ()
+ (with-temp-buffer
+ (insert "foo")
+ (buffer-string)))
+
+(defun comp-tests-lambda-return-f ()
+ (lambda (x) (1+ x)))
+
+(defun comp-tests-fib-f (n)
+ (cond ((= n 0) 0)
+ ((= n 1) 1)
+ (t (+ (comp-tests-fib-f (- n 1))
+ (comp-tests-fib-f (- n 2))))))
+
+(defmacro comp-tests-macro-m (x)
+ x)
+
+(defun comp-tests-string-trim-f (url)
+ (string-trim url))
+
+(defun comp-tests-trampoline-removal-f ()
+ (make-hash-table))
+
+(defun comp-tests-signal-f ()
+ (signal 'foo t))
+
+(defun comp-tests-func-call-removal-f ()
+ (let ((a 10)
+ (b 3))
+ (% a b)))
+
+\f
+;;;;;;;;;;;;;;;;;;;;
+;; Tromey's tests ;;
+;;;;;;;;;;;;;;;;;;;;
+
+;; Test Bconsp.
+(defun comp-test-consp (x) (consp x))
+
+;; Test Blistp.
+(defun comp-test-listp (x) (listp x))
+
+;; Test Bstringp.
+(defun comp-test-stringp (x) (stringp x))
+
+;; Test Bsymbolp.
+(defun comp-test-symbolp (x) (symbolp x))
+
+;; Test Bintegerp.
+(defun comp-test-integerp (x) (integerp x))
+
+;; Test Bnumberp.
+(defun comp-test-numberp (x) (numberp x))
+
+;; Test Badd1.
+(defun comp-test-add1 (x) (1+ x))
+
+;; Test Bsub1.
+(defun comp-test-sub1 (x) (1- x))
+
+;; Test Bneg.
+(defun comp-test-negate (x) (- x))
+
+;; Test Bnot.
+(defun comp-test-not (x) (not x))
+
+;; Test Bbobp, Beobp, Bpoint, Bpoint_min, Bpoint_max.
+(defun comp-test-bobp () (bobp))
+(defun comp-test-eobp () (eobp))
+(defun comp-test-point () (point))
+(defun comp-test-point-min () (point-min))
+(defun comp-test-point-max () (point-max))
+
+;; Test Bcar and Bcdr.
+(defun comp-test-car (x) (car x))
+(defun comp-test-cdr (x) (cdr x))
+
+;; Test Bcar_safe and Bcdr_safe.
+(defun comp-test-car-safe (x) (car-safe x))
+(defun comp-test-cdr-safe (x) (cdr-safe x))
+
+;; Test Beq.
+(defun comp-test-eq (x y) (eq x y))
+
+;; Test Bgotoifnil.
+(defun comp-test-if (x y) (if x x y))
+
+;; Test Bgotoifnilelsepop.
+(defun comp-test-and (x y) (and x y))
+
+;; Test Bgotoifnonnilelsepop.
+(defun comp-test-or (x y) (or x y))
+
+;; Test Bsave_excursion.
+(defun comp-test-save-excursion ()
+ (save-excursion
+ (insert "XYZ")))
+
+;; Test Bcurrent_buffer.
+(defun comp-test-current-buffer () (current-buffer))
+
+;; Test Bgtr.
+(defun comp-test-> (a b)
+ (> a b))
+
+;; Test Bpushcatch.
+(defun comp-test-catch (&rest l)
+ (catch 'done
+ (dolist (v l)
+ (when (> v 23)
+ (throw 'done v)))))
+
+;; Test Bmemq.
+(defun comp-test-memq (val list)
+ (memq val list))
+
+;; Test BlistN.
+(defun comp-test-listN (x)
+ (list x x x x x x x x x x x x x x x x))
+
+;; Test BconcatN.
+(defun comp-test-concatN (x)
+ (concat x x x x x x))
+
+;; Test optional and rest arguments.
+(defun comp-test-opt-rest (a &optional b &rest c)
+ (list a b c))
+
+;; Test for too many arguments.
+(defun comp-test-opt (a &optional b)
+ (cons a b))
+
+;; Test for unwind-protect.
+(defvar comp-test-up-val nil)
+(defun comp-test-unwind-protect (fun)
+ (setq comp-test-up-val nil)
+ (unwind-protect
+ (progn
+ (setq comp-test-up-val 23)
+ (funcall fun)
+ (setq comp-test-up-val 24))
+ (setq comp-test-up-val 999)))
+
+;; Non tested functions that proved just to be difficult to compile.
+
+(defun comp-test-callee (_ __) t)
+(defun comp-test-silly-frame1 (x)
+ ;; Check robustness against dead code.
+ (cl-case x
+ (0 (comp-test-callee
+ (pcase comp-tests-var1
+ (1 1)
+ (2 2))
+ 3))))
+
+(defun comp-test-silly-frame2 (token)
+ ;; Check robustness against dead code.
+ (while c
+ (cl-case c
+ (?< 1)
+ (?> 2))))
+
+(provide 'comp-test-funcs)
+
+;;; comp-test-funcs.el ends here
diff --git a/test/src/comp-tests.el b/test/src/comp-tests.el
new file mode 100644
index 0000000..570dcbd
--- /dev/null
+++ b/test/src/comp-tests.el
@@ -0,0 +1,487 @@
+;;; comp-tests.el --- unit tests for src/comp.c -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; Author: Andrea Corallo <akrl@sdf.org>
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Unit tests for src/comp.c.
+
+;;; Code:
+
+(require 'ert)
+(require 'cl-lib)
+(require 'comp)
+
+;; (setq comp-debug 1)
+(setq comp-speed 0)
+
+(defconst comp-test-directory (file-name-directory (or load-file-name
+ buffer-file-name)))
+(defconst comp-test-src
+ (concat comp-test-directory "comp-test-funcs.el"))
+
+(message "Compiling %s" comp-test-src)
+(load (native-compile comp-test-src))
+
+(ert-deftest comp-tests-bootstrap ()
+ "Compile the compiler and load it to compile it-self.
+Check that the resulting binaries do not differ."
+ (let* ((comp-src (concat comp-test-directory
+ "../../lisp/emacs-lisp/comp.el"))
+ (comp1-src (make-temp-file "stage1-" nil ".el"))
+ (comp2-src (make-temp-file "stage2-" nil ".el"))
+ (comp1 (concat comp1-src "n"))
+ (comp2 (concat comp2-src "n")))
+ (copy-file comp-src comp1-src t)
+ (copy-file comp-src comp2-src t)
+ (load (concat comp-src "c") nil nil t t)
+ (should (null (subr-native-elisp-p (symbol-function #'native-compile))))
+ (message "Compiling stage1...")
+ (load (native-compile comp1-src) nil nil t t)
+ (should (subr-native-elisp-p (symbol-function 'native-compile)))
+ (message "Compiling stage2...")
+ (native-compile comp2-src)
+ (message "Comparing %s %s" comp1 comp2)
+ (should (= (call-process "cmp" nil nil nil comp1 comp2) 0))))
+
+(ert-deftest comp-tests-provide ()
+ "Testing top level provide."
+ (should (featurep 'comp-test-funcs)))
+
+(ert-deftest comp-tests-varref ()
+ "Testing varref."
+ (should (= (comp-tests-varref-f) 3)))
+
+(ert-deftest comp-tests-list ()
+ "Testing cons car cdr."
+ (should (equal (comp-tests-list-f) '(1 2 3)))
+ (should (equal (comp-tests-list2-f 1 2 3) '(1 2 3)))
+ (should (= (comp-tests-car-f '(1 . 2)) 1))
+ (should (null (comp-tests-car-f nil)))
+ (should-error (comp-tests-car-f 3)
+ :type 'wrong-type-argument)
+ (should (= (comp-tests-cdr-f '(1 . 2)) 2))
+ (should (null (comp-tests-cdr-f nil)))
+ (should-error (comp-tests-cdr-f 3)
+ :type 'wrong-type-argument)
+ (should (= (comp-tests-car-safe-f '(1 . 2)) 1))
+ (should (null (comp-tests-car-safe-f 'a)))
+ (should (= (comp-tests-cdr-safe-f '(1 . 2)) 2))
+ (should (null (comp-tests-cdr-safe-f 'a))))
+
+(ert-deftest comp-tests-cons-car-cdr ()
+ "Testing cons car cdr."
+ (should (= (comp-tests-cons-car-f) 1))
+ (should (= (comp-tests-cons-cdr-f 3) 3)))
+
+(ert-deftest comp-tests-varset ()
+ "Testing varset."
+ (comp-tests-varset0-f)
+ (should (= comp-tests-var1 55))
+
+ (should (= (comp-tests-varset1-f) 4))
+ (should (= comp-tests-var1 66)))
+
+(ert-deftest comp-tests-length ()
+ "Testing length."
+ (should (= (comp-tests-length-f) 3)))
+
+(ert-deftest comp-tests-aref-aset ()
+ "Testing aref and aset."
+ (should (= (comp-tests-aref-aset-f) 100)))
+
+(ert-deftest comp-tests-symbol-value ()
+ "Testing aref and aset."
+ (should (= (comp-tests-symbol-value-f) 3)))
+
+(ert-deftest comp-tests-concat ()
+ "Testing concatX opcodes."
+ (should (string= (comp-tests-concat-f "bar") "abcdabcabfoobar")))
+
+(ert-deftest comp-tests-ffuncall ()
+ "Test calling conventions."
+
+ ;; (defun comp-tests-ffuncall-caller-f ()
+ ;; (comp-tests-ffuncall-callee-f 1 2 3))
+
+ ;; (should (equal (comp-tests-ffuncall-caller-f) '(1 2 3)))
+
+ ;; ;; After it gets compiled
+ ;; (native-compile #'comp-tests-ffuncall-callee-f)
+ ;; (should (equal (comp-tests-ffuncall-caller-f) '(1 2 3)))
+
+ ;; ;; Recompiling the caller once with callee already compiled
+ ;; (defun comp-tests-ffuncall-caller-f ()
+ ;; (comp-tests-ffuncall-callee-f 1 2 3))
+ ;; (should (equal (comp-tests-ffuncall-caller-f) '(1 2 3)))
+
+ (should (equal (comp-tests-ffuncall-callee-optional-f 1 2 3 4)
+ '(1 2 3 4)))
+ (should (equal (comp-tests-ffuncall-callee-optional-f 1 2 3)
+ '(1 2 3 nil)))
+ (should (equal (comp-tests-ffuncall-callee-optional-f 1 2)
+ '(1 2 nil nil)))
+
+ (should (equal (comp-tests-ffuncall-callee-rest-f 1 2)
+ '(1 2 nil)))
+ (should (equal (comp-tests-ffuncall-callee-rest-f 1 2 3)
+ '(1 2 (3))))
+ (should (equal (comp-tests-ffuncall-callee-rest-f 1 2 3 4)
+ '(1 2 (3 4))))
+
+ (should (equal (comp-tests-ffuncall-callee-more8-f 1 2 3 4 5 6 7 8 9 10)
+ '(1 2 3 4 5 6 7 8 9 10)))
+
+ (should (equal (comp-tests-ffuncall-callee-more8-rest-f 1 2 3 4 5 6 7 8 9 10 11)
+ '(1 2 3 4 5 6 7 8 9 (10 11))))
+
+ (should (equal (comp-tests-ffuncall-native-f) [nil]))
+
+ (should (equal (comp-tests-ffuncall-native-rest-f) [1 2 3]))
+
+ (should (equal (comp-tests-ffuncall-apply-many-f '(1 2 3))
+ '(1 2 3)))
+
+ (should (= (comp-tests-ffuncall-lambda-f 1) 2)))
+
+(ert-deftest comp-tests-jump-table ()
+ "Testing jump tables"
+ (should (eq (comp-tests-jump-table-1-f 'x) 'a))
+ (should (eq (comp-tests-jump-table-1-f 'y) 'b))
+ (should (eq (comp-tests-jump-table-1-f 'xxx) 'c))
+
+ ;; Jump table not with eq as test
+ (should (eq (comp-tests-jump-table-2-f "aaa") 'a))
+ (should (eq (comp-tests-jump-table-2-f "bbb") 'b)))
+
+(ert-deftest comp-tests-conditionals ()
+ "Testing conditionals."
+ (should (= (comp-tests-conditionals-1-f t) 1))
+ (should (= (comp-tests-conditionals-1-f nil) 2))
+ (should (= (comp-tests-conditionals-2-f t) 1340))
+ (should (eq (comp-tests-conditionals-2-f nil) nil)))
+
+(ert-deftest comp-tests-fixnum ()
+ "Testing some fixnum inline operation."
+ (should (= (comp-tests-fixnum-1-minus-f 10) 9))
+ (should (= (comp-tests-fixnum-1-minus-f most-negative-fixnum)
+ (1- most-negative-fixnum)))
+ (should-error (comp-tests-fixnum-1-minus-f 'a)
+ :type 'wrong-type-argument)
+ (should (= (comp-tests-fixnum-1-plus-f 10) 11))
+ (should (= (comp-tests-fixnum-1-plus-f most-positive-fixnum)
+ (1+ most-positive-fixnum)))
+ (should-error (comp-tests-fixnum-1-plus-f 'a)
+ :type 'wrong-type-argument)
+ (should (= (comp-tests-fixnum-minus-f 10) -10))
+ (should (= (comp-tests-fixnum-minus-f most-negative-fixnum)
+ (- most-negative-fixnum)))
+ (should-error (comp-tests-fixnum-minus-f 'a)
+ :type 'wrong-type-argument))
+
+(ert-deftest comp-tests-arith-comp ()
+ "Testing arithmetic comparisons."
+ (should (eq (comp-tests-eqlsign-f 4 3) nil))
+ (should (eq (comp-tests-eqlsign-f 3 3) t))
+ (should (eq (comp-tests-eqlsign-f 2 3) nil))
+ (should (eq (comp-tests-gtr-f 4 3) t))
+ (should (eq (comp-tests-gtr-f 3 3) nil))
+ (should (eq (comp-tests-gtr-f 2 3) nil))
+ (should (eq (comp-tests-lss-f 4 3) nil))
+ (should (eq (comp-tests-lss-f 3 3) nil))
+ (should (eq (comp-tests-lss-f 2 3) t))
+ (should (eq (comp-tests-les-f 4 3) nil))
+ (should (eq (comp-tests-les-f 3 3) t))
+ (should (eq (comp-tests-les-f 2 3) t))
+ (should (eq (comp-tests-geq-f 4 3) t))
+ (should (eq (comp-tests-geq-f 3 3) t))
+ (should (eq (comp-tests-geq-f 2 3) nil)))
+
+(ert-deftest comp-tests-setcarcdr ()
+ "Testing setcar setcdr."
+ (should (equal (comp-tests-setcar-f '(10 . 10) 3) '(3 . 10)))
+ (should (equal (comp-tests-setcdr-f '(10 . 10) 3) '(10 . 3)))
+ (should-error (comp-tests-setcar-f 3 10)
+ :type 'wrong-type-argument)
+ (should-error (comp-tests-setcdr-f 3 10)
+ :type 'wrong-type-argument))
+
+(ert-deftest comp-tests-bubble-sort ()
+ "Run bubble sort."
+ (let* ((list1 (mapcar #'random (make-list 1000 most-positive-fixnum)))
+ (list2 (copy-sequence list1)))
+ (should (equal (comp-bubble-sort-f list1)
+ (sort list2 #'<)))))
+
+(ert-deftest comp-test-apply ()
+ "Test some inlined list functions."
+ (should (eq (comp-tests-consp-f '(1)) t))
+ (should (eq (comp-tests-consp-f 1) nil))
+ (let ((x (cons 1 2)))
+ (should (= (comp-tests-setcar2-f x) 3))
+ (should (equal x '(3 . 2)))))
+
+(ert-deftest comp-tests-num-inline ()
+ "Test some inlined number functions."
+ (should (eq (comp-tests-integerp-f 1) t))
+ (should (eq (comp-tests-integerp-f '(1)) nil))
+ (should (eq (comp-tests-integerp-f 3.5) nil))
+ (should (eq (comp-tests-integerp-f (1+ most-negative-fixnum)) t))
+
+ (should (eq (comp-tests-numberp-f 1) t))
+ (should (eq (comp-tests-numberp-f 'a) nil))
+ (should (eq (comp-tests-numberp-f 3.5) t)))
+
+(ert-deftest comp-tests-stack ()
+ "Test some stack operation."
+ (should (= (comp-tests-discardn-f 10) 2))
+ (should (string= (with-temp-buffer
+ (comp-tests-insertn-f "a" "b" "c" "d")
+ (buffer-string))
+ "abcd")))
+
+(ert-deftest comp-tests-non-locals ()
+ "Test non locals."
+ (let ((gc-cons-threshold most-positive-fixnum)) ;; FIXME!!
+ (should (string= (comp-tests-condition-case-0-f)
+ "arith-error Arithmetic error catched"))
+ (should (string= (comp-tests-condition-case-1-f)
+ "error foo catched"))
+ (should (= (comp-tests-catch-f
+ (lambda () (throw 'foo 3)))
+ 3))
+ (should (= (catch 'foo
+ (comp-tests-throw-f 3))))))
+
+(ert-deftest comp-tests-gc ()
+ "Try to do some longer computation to let the gc kick in."
+ (dotimes (_ 100000)
+ (comp-tests-cons-cdr-f 3))
+ (should (= (comp-tests-cons-cdr-f 3) 3)))
+
+(ert-deftest comp-tests-buffer ()
+ (should (string= (comp-tests-buff0-f) "foo")))
+
+(ert-deftest comp-tests-lambda-return ()
+ (should (= (funcall (comp-tests-lambda-return-f) 3) 4)))
+
+(ert-deftest comp-tests-recursive ()
+ (should (= (comp-tests-fib-f 10) 55)))
+
+(ert-deftest comp-tests-macro ()
+ "Just check we can define macros"
+ (should (macrop (symbol-function 'comp-tests-macro-m))))
+
+(ert-deftest comp-tests-string-trim ()
+ (should (string= (comp-tests-string-trim-f "dsaf ") "dsaf")))
+
+(ert-deftest comp-tests-trampoline-removal ()
+ ;; This tests that we can can call primitives with no dedicated bytecode.
+ ;; At speed >= 2 the trampoline will not be used.
+ (should (hash-table-p (comp-tests-trampoline-removal-f))))
+
+(ert-deftest comp-tests-signal ()
+ (should (equal (condition-case err
+ (comp-tests-signal-f)
+ (t err))
+ '(foo . t))))
+
+(ert-deftest comp-tests-func-call-removal ()
+ ;; See `comp-propagate-insn' `comp-function-call-remove'.
+ (should (= (comp-tests-func-call-removal-f) 1)))
+
+(ert-deftest comp-tests-free-fun ()
+ "Check we are able to compile a single function."
+ (defun comp-tests-free-fun-f ()
+ 3)
+ (load (native-compile #'comp-tests-free-fun-f))
+ (should (subr-native-elisp-p (symbol-function #'comp-tests-free-fun-f)))
+ (should (= (comp-tests-free-fun-f) 3)))
+
+\f
+;;;;;;;;;;;;;;;;;;;;
+;; Tromey's tests ;;
+;;;;;;;;;;;;;;;;;;;;
+
+(ert-deftest comp-consp ()
+ (should-not (comp-test-consp 23))
+ (should-not (comp-test-consp nil))
+ (should (comp-test-consp '(1 . 2))))
+
+(ert-deftest comp-listp ()
+ (should-not (comp-test-listp 23))
+ (should (comp-test-listp nil))
+ (should (comp-test-listp '(1 . 2))))
+
+(ert-deftest comp-stringp ()
+ (should-not (comp-test-stringp 23))
+ (should-not (comp-test-stringp nil))
+ (should (comp-test-stringp "hi")))
+
+(ert-deftest comp-symbolp ()
+ (should-not (comp-test-symbolp 23))
+ (should-not (comp-test-symbolp "hi"))
+ (should (comp-test-symbolp 'whatever)))
+
+(ert-deftest comp-integerp ()
+ (should (comp-test-integerp 23))
+ (should-not (comp-test-integerp 57.5))
+ (should-not (comp-test-integerp "hi"))
+ (should-not (comp-test-integerp 'whatever)))
+
+(ert-deftest comp-numberp ()
+ (should (comp-test-numberp 23))
+ (should (comp-test-numberp 57.5))
+ (should-not (comp-test-numberp "hi"))
+ (should-not (comp-test-numberp 'whatever)))
+
+(ert-deftest comp-add1 ()
+ (should (eq (comp-test-add1 23) 24))
+ (should (eq (comp-test-add1 -17) -16))
+ (should (eql (comp-test-add1 1.0) 2.0))
+ (should-error (comp-test-add1 nil)
+ :type 'wrong-type-argument))
+
+(ert-deftest comp-sub1 ()
+ (should (eq (comp-test-sub1 23) 22))
+ (should (eq (comp-test-sub1 -17) -18))
+ (should (eql (comp-test-sub1 1.0) 0.0))
+ (should-error (comp-test-sub1 nil)
+ :type 'wrong-type-argument))
+
+(ert-deftest comp-negate ()
+ (should (eq (comp-test-negate 23) -23))
+ (should (eq (comp-test-negate -17) 17))
+ (should (eql (comp-test-negate 1.0) -1.0))
+ (should-error (comp-test-negate nil)
+ :type 'wrong-type-argument))
+
+(ert-deftest comp-not ()
+ (should (eq (comp-test-not 23) nil))
+ (should (eq (comp-test-not nil) t))
+ (should (eq (comp-test-not t) nil)))
+
+(ert-deftest comp-bobp-and-eobp ()
+ (with-temp-buffer
+ (should (comp-test-bobp))
+ (should (comp-test-eobp))
+ (insert "hi")
+ (goto-char (point-min))
+ (should (eq (comp-test-point-min) (point-min)))
+ (should (eq (comp-test-point) (point-min)))
+ (should (comp-test-bobp))
+ (should-not (comp-test-eobp))
+ (goto-char (point-max))
+ (should (eq (comp-test-point-max) (point-max)))
+ (should (eq (comp-test-point) (point-max)))
+ (should-not (comp-test-bobp))
+ (should (comp-test-eobp))))
+
+(ert-deftest comp-car-cdr ()
+ (let ((pair '(1 . b)))
+ (should (eq (comp-test-car pair) 1))
+ (should (eq (comp-test-car nil) nil))
+ (should-error (comp-test-car 23)
+ :type 'wrong-type-argument)
+ (should (eq (comp-test-cdr pair) 'b))
+ (should (eq (comp-test-cdr nil) nil))
+ (should-error (comp-test-cdr 23)
+ :type 'wrong-type-argument)))
+
+(ert-deftest comp-car-cdr-safe ()
+ (let ((pair '(1 . b)))
+ (should (eq (comp-test-car-safe pair) 1))
+ (should (eq (comp-test-car-safe nil) nil))
+ (should (eq (comp-test-car-safe 23) nil))
+ (should (eq (comp-test-cdr-safe pair) 'b))
+ (should (eq (comp-test-cdr-safe nil) nil))
+ (should (eq (comp-test-cdr-safe 23) nil))))
+
+(ert-deftest comp-eq ()
+ (should (comp-test-eq 'a 'a))
+ (should (comp-test-eq 5 5))
+ (should-not (comp-test-eq 'a 'b))
+ (should-not (comp-test-eq "x" "x")))
+
+(ert-deftest comp-if ()
+ (should (eq (comp-test-if 'a 'b) 'a))
+ (should (eq (comp-test-if 0 23) 0))
+ (should (eq (comp-test-if nil 'b) 'b)))
+
+(ert-deftest comp-and ()
+ (should (eq (comp-test-and 'a 'b) 'b))
+ (should (eq (comp-test-and 0 23) 23))
+ (should (eq (comp-test-and nil 'b) nil)))
+
+(ert-deftest comp-or ()
+ (should (eq (comp-test-or 'a 'b) 'a))
+ (should (eq (comp-test-or 0 23) 0))
+ (should (eq (comp-test-or nil 'b) 'b)))
+
+(ert-deftest comp-save-excursion ()
+ (with-temp-buffer
+ (comp-test-save-excursion)
+ (should (eq (point) (point-min)))
+ (should (eq (comp-test-current-buffer) (current-buffer)))))
+
+(ert-deftest comp-> ()
+ (should (eq (comp-test-> 0 23) nil))
+ (should (eq (comp-test-> 23 0) t)))
+
+(ert-deftest comp-catch ()
+ (should (eq (comp-test-catch 0 1 2 3 4) nil))
+ (should (eq (comp-test-catch 20 21 22 23 24 25 26 27 28) 24)))
+
+(ert-deftest comp-memq ()
+ (should (equal (comp-test-memq 0 '(5 4 3 2 1 0)) '(0)))
+ (should (eq (comp-test-memq 72 '(5 4 3 2 1 0)) nil)))
+
+(ert-deftest comp-listN ()
+ (should (equal (comp-test-listN 57)
+ '(57 57 57 57 57 57 57 57 57 57 57 57 57 57 57 57))))
+
+(ert-deftest comp-concatN ()
+ (should (equal (comp-test-concatN "x") "xxxxxx")))
+
+(ert-deftest comp-opt-rest ()
+ (should (equal (comp-test-opt-rest 1) '(1 nil nil)))
+ (should (equal (comp-test-opt-rest 1 2) '(1 2 nil)))
+ (should (equal (comp-test-opt-rest 1 2 3) '(1 2 (3))))
+ (should (equal (comp-test-opt-rest 1 2 56 57 58)
+ '(1 2 (56 57 58)))))
+
+(ert-deftest comp-opt ()
+ (should (equal (comp-test-opt 23) '(23)))
+ (should (equal (comp-test-opt 23 24) '(23 . 24)))
+ (should-error (comp-test-opt)
+ :type 'wrong-number-of-arguments)
+ (should-error (comp-test-opt nil 24 97)
+ :type 'wrong-number-of-arguments))
+
+(ert-deftest comp-unwind-protect ()
+ (comp-test-unwind-protect 'ignore)
+ (should (eq comp-test-up-val 999))
+ (condition-case nil
+ (comp-test-unwind-protect (lambda () (error "HI")))
+ (error
+ nil))
+ (should (eq comp-test-up-val 999)))
+
+;;; comp-tests.el ends here
^ permalink raw reply related [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-11-28 6:54 On elisp running native Andrea Corallo
@ 2019-11-28 11:31 ` Óscar Fuentes
2019-11-28 14:07 ` Andrea Corallo
2019-11-28 12:47 ` Lars Ingebrigtsen
` (3 subsequent siblings)
4 siblings, 1 reply; 210+ messages in thread
From: Óscar Fuentes @ 2019-11-28 11:31 UTC (permalink / raw)
To: emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> Hi all. I'd like share the investigation I've been working on to give
> Emacs the ability to compile and run elisp as native code.
>
> Despite being in very early stage it's functional and shows some quite
> interesting potential in terms of performance benefit. I then thought
> was at a good stage to be shared and discussed.
>
> I wrote a small page to describe what I did and how it works plus the
> current status not to have the information spread in various mails.
>
> http://akrl.sdf.org/gccemacs.html
nbody is ~3.5x faster with
No, it is ~4.6 faster :-)
Please consider using factors instead of % for indicating the speed
boost on the table. Percentages are confusing.
One interesting topic is how your approach affects the features that
characterizes Emacs, such as the ability to redefine almost anything at
runtime, instrospection, debugging, etc.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-11-28 11:31 ` Óscar Fuentes
@ 2019-11-28 14:07 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-11-28 14:07 UTC (permalink / raw)
To: Óscar Fuentes; +Cc: emacs-devel
Óscar Fuentes <ofv@wanadoo.es> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> Hi all. I'd like share the investigation I've been working on to give
>> Emacs the ability to compile and run elisp as native code.
>>
>> Despite being in very early stage it's functional and shows some quite
>> interesting potential in terms of performance benefit. I then thought
>> was at a good stage to be shared and discussed.
>>
>> I wrote a small page to describe what I did and how it works plus the
>> current status not to have the information spread in various mails.
>>
>> http://akrl.sdf.org/gccemacs.html
>
> nbody is ~3.5x faster with
>
> No, it is ~4.6 faster :-)
>
> Please consider using factors instead of % for indicating the speed
> boost on the table. Percentages are confusing.
Ops... Apparently this was sufficently confusing to confuse my self :)
Should be fixed by now.
>
> One interesting topic is how your approach affects the features that
> characterizes Emacs, such as the ability to redefine almost anything at
> runtime, instrospection, debugging, etc.
>
Good points:
* Debuggability
What you get from native compiled code should be more or less what
you get from the byte-compiled one.
The caveat is that at speed >= 2 currently I'm optimizing out the
funcall trampoline for calling primitive C functions. This should
impact the back-trace accuracy.
A possible solution would be to emit some native code to record that
before each call. Another one is not to perform this optimization
at all. This is a trade-off.
* Introspection
As for the previous point this should be the same of what you get with
the byte-compiled code (once I get the doc working).
* Ability redefine to functions at run-time
Nothing should change up to speed 2 here. At speed 3 the compiler
can in-line within the compilation unit so you have to recompile it
all.
I see speed 3 like the setting where you allow a bit more extreme
optimizations to be performed. Something to be used not often and
with extreme care.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 6:54 On elisp running native Andrea Corallo
2019-11-28 11:31 ` Óscar Fuentes
@ 2019-11-28 12:47 ` Lars Ingebrigtsen
2019-11-28 20:15 ` Andrea Corallo
2019-11-28 16:04 ` Stefan Monnier
` (2 subsequent siblings)
4 siblings, 1 reply; 210+ messages in thread
From: Lars Ingebrigtsen @ 2019-11-28 12:47 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> I wrote a small page to describe what I did and how it works plus the
> current status not to have the information spread in various mails.
>
> http://akrl.sdf.org/gccemacs.html
Wow, that's exciting!
A couple of questions:
> * Native compiler is not re-entrant:
>
> Just top level functions are native compiled, the others (lambda
> included) are still kept as byte-code.
Is that a fundamental limitation, or have you just not gotten around to
it yet?
Also, I'm wondering about debugging: I spend a lot of time in Emacs just
edebugging or in the *Backtrace* buffer. Does the natively compiled
code mean that all that has to happen in gdb now?
--
(domestic pets only, the antidote for overdose, milk.)
bloggy blog: http://lars.ingebrigtsen.no
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-11-28 12:47 ` Lars Ingebrigtsen
@ 2019-11-28 20:15 ` Andrea Corallo
2019-11-29 6:03 ` Lars Brinkhoff
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-11-28 20:15 UTC (permalink / raw)
To: Lars Ingebrigtsen; +Cc: emacs-devel
Lars Ingebrigtsen <larsi@gnus.org> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> I wrote a small page to describe what I did and how it works plus the
>> current status not to have the information spread in various mails.
>>
>> http://akrl.sdf.org/gccemacs.html
>
> Wow, that's exciting!
>
> A couple of questions:
>
>> * Native compiler is not re-entrant:
>>
>> Just top level functions are native compiled, the others (lambda
>> included) are still kept as byte-code.
>
> Is that a fundamental limitation, or have you just not gotten around to
> it yet?
I do not see any design limitation for that. Just had no time to work
on it.
> Also, I'm wondering about debugging: I spend a lot of time in Emacs just
> edebugging or in the *Backtrace* buffer. Does the natively compiled
> code mean that all that has to happen in gdb now?
No, you can trap into native compiled code with a gdb as you can do it
for any piece of the C core, but I would do that just for debugging the
internals or the functionality of the generated code. Should be
something not routinely done for writing lisp.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 20:15 ` Andrea Corallo
@ 2019-11-29 6:03 ` Lars Brinkhoff
0 siblings, 0 replies; 210+ messages in thread
From: Lars Brinkhoff @ 2019-11-29 6:03 UTC (permalink / raw)
To: emacs-devel
Hello Andrea,
First off, I think this is very exciting news and possibly a major step
forward for Emacs. So thank you.
Andrea Corallo wrote:
> Lars Ingebrigtsen wrote:
>> Also, I'm wondering about debugging: I spend a lot of time in Emacs just
>> edebugging or in the *Backtrace* buffer. Does the natively compiled
>> code mean that all that has to happen in gdb now?
> No, you can trap into native compiled code with a gdb as you can do it
> for any piece of the C core, but I would do that just for debugging the
> internals or the functionality of the generated code. Should be
> something not routinely done for writing lisp.
I think the concern is whether natively compiled code would be less
debuggable. And I think the answer would be: it depends. If speed is
set to 3, maybe you forego some debugging features. And if it's low
enough, debugging would hopefully work exactly the same as byte-compiled
code. The default for building Emacs might be somewhere in between but
probably lean more towards debugging.
Are there more compiler parameters than speed, like debug, safety, space
as in CL?
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 6:54 On elisp running native Andrea Corallo
2019-11-28 11:31 ` Óscar Fuentes
2019-11-28 12:47 ` Lars Ingebrigtsen
@ 2019-11-28 16:04 ` Stefan Monnier
2019-11-28 17:28 ` Andrea Corallo
2019-12-08 20:18 ` Andrea Corallo
2020-03-15 14:21 ` Spenser Truex
4 siblings, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2019-11-28 16:04 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> Hi all. I'd like share the investigation I've been working on to give
> Emacs the ability to compile and run elisp as native code.
This looks great, thanks.
[ I wasn't able to follow all the explanations at
http://akrl.sdf.org/gccemacs.html, such as the one around "function
frames", with which I'm not familiar. Are these like activation
frames? ]
A few questions:
- How did you get there? I see some "we" in the web page, which makes
it sound like you weren't completely alone.
- AFAICT most of the new compilation code is in comp.el and comp.c focuses
on "code generation", right? Can you say something about the time it
takes to compile files?
What does the time break down looks like (i.e. percentage spent in
bytecompiler, comp.el, comp.c, libgccjit)?
- Have you tried to use the compiler as benchmark (i.e. how much faster
can Emacs compile (either byte-compile or native-compile)) if the
compiler code is native-compiled (since it's all using
lexical-binding already)?
-- Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 16:04 ` Stefan Monnier
@ 2019-11-28 17:28 ` Andrea Corallo
2019-11-28 21:06 ` Stefan Monnier
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-11-28 17:28 UTC (permalink / raw)
To: Stefan Monnier; +Cc: emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>> Hi all. I'd like share the investigation I've been working on to give
>> Emacs the ability to compile and run elisp as native code.
>
> This looks great, thanks.
>
> [ I wasn't able to follow all the explanations at
> http://akrl.sdf.org/gccemacs.html, such as the one around "function
> frames", with which I'm not familiar. Are these like activation
> frames? ]
Yes I think we both mean the same. In this case basically where we store
automatic variables and data related to the single activated function.
> A few questions:
>
> - How did you get there? I see some "we" in the web page, which makes
> it sound like you weren't completely alone.
Sorry for that I'm not much into using 'I'. I actually did the design
and all the coding my self, but I've been helped with moral support and
with the testing by two brave colleagues (one contributed to the elisp
benchmarks too porting dhrystone).
> - AFAICT most of the new compilation code is in comp.el and comp.c focuses
> on "code generation", right? Can you say something about the time it
> takes to compile files?
> What does the time break down looks like (i.e. percentage spent in
> bytecompiler, comp.el, comp.c, libgccjit)?
All measurable time is spent in gcc from my early measures. The time
spent in comp.el seems really a rounding error.
> - Have you tried to use the compiler as benchmark (i.e. how much faster
> can Emacs compile (either byte-compile or native-compile)) if the
> compiler code is native-compiled (since it's all using
> lexical-binding already)?
I use the compiler native compiled but because of the previous point I
think is hard to measure the difference.
Talking about compile time in general I think we are looking at
something like few minutes to compile the whole Emacs at speed 0. The
time goes up to say ~4 hours with 4 cores for the same job at speed 2.
I think it will be interesting to look into the gcc compilation pipe to
see where we are losing so much time, my guess is that there's one or
few passes that go a little nuts with all the moves we do. I had no
time to look into it but my guess is that once understood the problem we
can probably dime it down.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 17:28 ` Andrea Corallo
@ 2019-11-28 21:06 ` Stefan Monnier
2019-11-28 22:52 ` Andrea Corallo
` (2 more replies)
0 siblings, 3 replies; 210+ messages in thread
From: Stefan Monnier @ 2019-11-28 21:06 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
>> [ I wasn't able to follow all the explanations at
>> http://akrl.sdf.org/gccemacs.html, such as the one around "function
>> frames", with which I'm not familiar. Are these like activation
>> frames? ]
> Yes I think we both mean the same. In this case basically where we store
> automatic variables and data related to the single activated function.
OK, thanks. I think "activation frame" is the standard term (of which
there can be several on the stack at the same time in case of recursion).
>> - How did you get there? I see some "we" in the web page, which makes
>> it sound like you weren't completely alone.
> Sorry for that I'm not much into using 'I'.
That's OK, but I see you tried to use it as a clever ploy to dodge the
initial question: how did you get there?
>> - Have you tried to use the compiler as benchmark (i.e. how much faster
>> can Emacs compile (either byte-compile or native-compile)) if the
>> compiler code is native-compiled (since it's all using
>> lexical-binding already)?
> I use the compiler native compiled but because of the previous point I
> think is hard to measure the difference.
How 'bout measuring the time to byte-compile a given set of files, then:
first using the byte-compiled compiler and then using the
native-compiled compiler (where "compiler" here means at least cconv.el,
byte-opt.el, bytecomp.el, and macroexp.el)?
BTW, I think developing a good set of Elisp benchmarks is useful
independently from this, so I'd encourage you to submit your benchmarks
as a new GNU ELPA package (we could also incorporate it into Emacs
itself, but I think we'll want to use it to compare performance between
diverse Emacsen, so a separate package makes more sense).
Maybe someone from the Gnus side will want to submit more benchmarks
(such as one that manipulates "sets/ranges or article numbers").
> Talking about compile time in general I think we are looking at
> something like few minutes to compile the whole Emacs at speed 0. The
> time goes up to say ~4 hours with 4 cores for the same job at speed 2.
[ Compile time varies for me with the normal Emacs from less than
5 minutes to more than an hour depending on the machine on which
I compile, so absolute times don't speak to me very much. ]
So, IIUC, with enough optimisations enabled, we gets into "a long
time" territory?
> I think it will be interesting to look into the gcc compilation pipe to
> see where we are losing so much time, my guess is that there's one or
> few passes that go a little nuts with all the moves we do. I had no
> time to look into it but my guess is that once understood the problem we
> can probably dime it down.
Indeed, I'm surprised that compilation time in gcc would blow up by
significantly more than a factor 10 just because of optimisation
options, so either we're using optimisations which are really too
costly, or there should be something we can do to avoid this blow up
without any significant performance loss.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-11-28 21:06 ` Stefan Monnier
@ 2019-11-28 22:52 ` Andrea Corallo
2019-11-29 0:20 ` Eric Abrahamsen
2019-11-29 14:59 ` Andrea Corallo
2 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-11-28 22:52 UTC (permalink / raw)
To: Stefan Monnier; +Cc: emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>>> [ I wasn't able to follow all the explanations at
>>> http://akrl.sdf.org/gccemacs.html, such as the one around "function
>>> frames", with which I'm not familiar. Are these like activation
>>> frames? ]
>> Yes I think we both mean the same. In this case basically where we store
>> automatic variables and data related to the single activated function.
>
> OK, thanks. I think "activation frame" is the standard term (of which
> there can be several on the stack at the same time in case of recursion).
OK good to know, I don't have an official education as a compiler engineer
so my nomenclature can be occasionally a bit off.
>>> - How did you get there? I see some "we" in the web page, which makes
>>> it sound like you weren't completely alone.
>> Sorry for that I'm not much into using 'I'.
>
> That's OK, but I see you tried to use it as a clever ploy to dodge the
> initial question: how did you get there?
Sorry I though the question was more to understand if there were other
contributors for copyright reasons.
On my side the long story short is about this:
I was already into gcc and libgccjit. I thought was cool to apply these
to some lisp implementation. I decided to have Emacs as a target cause
I imagined would have been useful and because I'm obviously an Emacs
user and fan.
I wanted to do something with the potential to be completed and
up streamed one day. Therefore I discarded the idea of writing the full
lisp front-end from scratch. On the other side I considered the idea
seen in previous projects of reusing the byte-compiler infrastructure
quite clever.
The original plan was the to do something like Tromey's jitter but gcc
based and with a mechanism to reload the compiled code. So I did it.
I had a single pass compiler all written in C that was decoding the byte
code and driving libgccjit.
I was quite unhappy with that solution for two reasons:
1- The C file was getting huge without doing anything really smart.
2- After some test and observation became clear that to generate
efficient code this approach was quite limited and a more sophisticated
approach with a propagation engine and the classical compiler theory
data structures was needed. The idea of just driving gcc and having
everything magically optimized was simply naive.
So I came up with the idea of defining LIMPLE and using it as interface
between the C and the lisp side of the compiler.
In this way I had the right IR for implementing the 'clever' algorithmic
into lisp and the C side has just to 'replay' the result on libgccjit.
Moreover it saved me from the pain of exposing libgccjit to lisp.
I then realized I could, instead of decoding op-codes, just spill the LAP
from the byte-compiler. This makes the system simpler and more robust
cause I get also information on the stack depth I can double check or
use during limplification.
Lastly I managed to reuse the information defined in the byte-compiler
on the stack offset of every op to generate automatically or semi the
code of my compiler for the translation from LAP to LIMPLE for good part
of the op codes.
The rest just iterating over tests debugging and implementing.
I'm not sure this answer the question. Does it?
>>> - Have you tried to use the compiler as benchmark (i.e. how much faster
>>> can Emacs compile (either byte-compile or native-compile)) if the
>>> compiler code is native-compiled (since it's all using
>>> lexical-binding already)?
>> I use the compiler native compiled but because of the previous point I
>> think is hard to measure the difference.
>
> How 'bout measuring the time to byte-compile a given set of files, then:
> first using the byte-compiled compiler and then using the
> native-compiled compiler (where "compiler" here means at least cconv.el,
> byte-opt.el, bytecomp.el, and macroexp.el)?
I think is a test we can do, sounds like a very good benchmark.
> BTW, I think developing a good set of Elisp benchmarks is useful
> independently from this, so I'd encourage you to submit your benchmarks
> as a new GNU ELPA package (we could also incorporate it into Emacs
> itself, but I think we'll want to use it to compare performance between
> diverse Emacsen, so a separate package makes more sense).
OK I'll do some clean-up then. BTW I think for dhrystone my colleague
has to do the paper-works and we have to look also into the original
license.
> Maybe someone from the Gnus side will want to submit more benchmarks
> (such as one that manipulates "sets/ranges or article numbers").
>
>> Talking about compile time in general I think we are looking at
>> something like few minutes to compile the whole Emacs at speed 0. The
>> time goes up to say ~4 hours with 4 cores for the same job at speed 2.
>
> [ Compile time varies for me with the normal Emacs from less than
> 5 minutes to more than an hour depending on the machine on which
> I compile, so absolute times don't speak to me very much. ]
>
> So, IIUC, with enough optimisations enabled, we gets into "a long
> time" territory?
Yes feels like compiling C++ :x
>> I think it will be interesting to look into the gcc compilation pipe to
>> see where we are losing so much time, my guess is that there's one or
>> few passes that go a little nuts with all the moves we do. I had no
>> time to look into it but my guess is that once understood the problem we
>> can probably dime it down.
>
> Indeed, I'm surprised that compilation time in gcc would blow up by
> significantly more than a factor 10 just because of optimisation
> options, so either we're using optimisations which are really too
> costly, or there should be something we can do to avoid this blow up
> without any significant performance loss.
Me too. My guess is that because the code we feed into gcc does not
look at all like it's written by a human something has just to be tuned
somewhere in some pass. I'm looking forward to investigate it but I'm a
bit saturated now.
I'm quite confident we can at least mitigate it but in general maybe we
should give also the possibility to specify different optimization
levels per function granularity.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 21:06 ` Stefan Monnier
2019-11-28 22:52 ` Andrea Corallo
@ 2019-11-29 0:20 ` Eric Abrahamsen
2019-11-29 14:59 ` Andrea Corallo
2 siblings, 0 replies; 210+ messages in thread
From: Eric Abrahamsen @ 2019-11-29 0:20 UTC (permalink / raw)
To: emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>>> [ I wasn't able to follow all the explanations at
>>> http://akrl.sdf.org/gccemacs.html, such as the one around "function
>>> frames", with which I'm not familiar. Are these like activation
>>> frames? ]
>> Yes I think we both mean the same. In this case basically where we store
>> automatic variables and data related to the single activated function.
>
> OK, thanks. I think "activation frame" is the standard term (of which
> there can be several on the stack at the same time in case of recursion).
>
>>> - How did you get there? I see some "we" in the web page, which makes
>>> it sound like you weren't completely alone.
>> Sorry for that I'm not much into using 'I'.
>
> That's OK, but I see you tried to use it as a clever ploy to dodge the
> initial question: how did you get there?
>
>>> - Have you tried to use the compiler as benchmark (i.e. how much faster
>>> can Emacs compile (either byte-compile or native-compile)) if the
>>> compiler code is native-compiled (since it's all using
>>> lexical-binding already)?
>> I use the compiler native compiled but because of the previous point I
>> think is hard to measure the difference.
>
> How 'bout measuring the time to byte-compile a given set of files, then:
> first using the byte-compiled compiler and then using the
> native-compiled compiler (where "compiler" here means at least cconv.el,
> byte-opt.el, bytecomp.el, and macroexp.el)?
>
> BTW, I think developing a good set of Elisp benchmarks is useful
> independently from this, so I'd encourage you to submit your benchmarks
> as a new GNU ELPA package (we could also incorporate it into Emacs
> itself, but I think we'll want to use it to compare performance between
> diverse Emacsen, so a separate package makes more sense).
>
> Maybe someone from the Gnus side will want to submit more benchmarks
> (such as one that manipulates "sets/ranges or article numbers").
Maybe so!
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 21:06 ` Stefan Monnier
2019-11-28 22:52 ` Andrea Corallo
2019-11-29 0:20 ` Eric Abrahamsen
@ 2019-11-29 14:59 ` Andrea Corallo
2019-11-29 15:10 ` Eli Zaretskii
2 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-11-29 14:59 UTC (permalink / raw)
To: Stefan Monnier; +Cc: emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> How 'bout measuring the time to byte-compile a given set of files, then:
> first using the byte-compiled compiler and then using the
> native-compiled compiler (where "compiler" here means at least cconv.el,
> byte-opt.el, bytecomp.el, and macroexp.el)?
I did the following simple test.
I byte-compiled all files in emacs/lisp (avoiding sub-folders). Using
two Emacs, one conventionally byte-compiled the other one after having
loaded all native compiled files.
Note that in this case all lisp files were native compiled at *speed 2*.
The result of this very preliminary test looks like this. Please take
this with grain of salt:
| | byte-compiled | native-compiled | speed-up |
|-------------+---------------+-----------------+----------|
| interactive | 27s | 19s | 1.4x |
| batch | 15s | 8s | 1.9x |
To me makes quite sense that in interactive there's less difference
cause of the re-display done in C anyway.
To a quick look with perf seems garbage collection is also a
considerable part of the result.
I think is encouraging and I'd really like having somebody reproducing
these kind of measure.
I'll have more time to look into all of this in the week-end.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-29 14:59 ` Andrea Corallo
@ 2019-11-29 15:10 ` Eli Zaretskii
2019-11-29 15:30 ` Stefan Monnier
2019-11-29 16:43 ` Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-11-29 15:10 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Fri, 29 Nov 2019 14:59:16 +0000
> Cc: emacs-devel@gnu.org
>
> | | byte-compiled | native-compiled | speed-up |
> |-------------+---------------+-----------------+----------|
> | interactive | 27s | 19s | 1.4x |
> | batch | 15s | 8s | 1.9x |
>
> To me makes quite sense that in interactive there's less difference
> cause of the re-display done in C anyway.
Regardless of the speed-up due to using native code, it sounds strange
to me that byte-compiling interactively takes twice the time it takes
non-interactively. I have hard time convincing myself that redisplay
has such a profound effect on byte compilation, since byte-compiling a
file normally yields one or two lines of output, regardless of the
size of the file being compiled.
Does the timing change if you delete the window showing the
compilation log? In that case, redisplay is not involved at all.
Also, was the interactive session started only once, and then used for
compiling all the files? Or was it restarted anew for each
compilation? If the latter, then the slow-down is probably due to a
slower startup of a GUI session.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-29 15:10 ` Eli Zaretskii
@ 2019-11-29 15:30 ` Stefan Monnier
2019-12-01 10:34 ` Andrea Corallo
2019-11-29 16:43 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2019-11-29 15:30 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel, Andrea Corallo
>> | | byte-compiled | native-compiled | speed-up |
>> |-------------+---------------+-----------------+----------|
>> | interactive | 27s | 19s | 1.4x |
>> | batch | 15s | 8s | 1.9x |
So, there's 11s-12s of "interactive cost".
>> To me makes quite sense that in interactive there's less difference
>> cause of the re-display done in C anyway.
I'm not sure what those 11s-12s come from, but I think it's safe to take
the "1.9x" as the more relevant measure in terms of measuring the impact
of native compilation on the speed of bytecomp.el.
> Regardless of the speed-up due to using native code, it sounds strange
> to me that byte-compiling interactively takes twice the time it takes
> non-interactively. I have hard time convincing myself that redisplay
> has such a profound effect on byte compilation, since byte-compiling a
> file normally yields one or two lines of output, regardless of the
> size of the file being compiled.
I agree it's a bit surprising. Even it was running in a "large"
interactive session, it could also be due to the time it takes to GC
a larger heap (tho we normally GC such larger heaps proportionally less
often, so that doesn't really explain it either).
But since this is orthogonal to the gccemacs native compilation, maybe
we should move it to a bug report where we can try and investigate it.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-11-29 15:30 ` Stefan Monnier
@ 2019-12-01 10:34 ` Andrea Corallo
2019-12-01 15:27 ` Stefan Monnier
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-01 10:34 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> I'm not sure what those 11s-12s come from, but I think it's safe to take
> the "1.9x" as the more relevant measure in terms of measuring the impact
> of native compilation on the speed of bytecomp.el.
I've got some time to repeat the measure trying to understand better
what's going on.
I've repeated the same test but running it 10 times.
(benchmark 10 '(mapc #'byte-compile-file (directory-files
"~/emacs/lisp" t "el$")))
I've discovered that my previous measure made not much sense because the
time spent into GC is much bigger during the byte-code runs.
-bytecode
"Elapsed time: 156.960728s (77.049537s in 3421 GCs)"
-native speed 2
"Elapsed time: 94.538094s (46.362582s in 1737 GCs)"
-native speed 3
"Elapsed time: 92.288413s (47.509687s in 1816 GCs)"
My explanation for that (you'll tell me if it makes sense) is that
having to load by hand all the .eln we start the test native compiled
tests with a bigger heap:
(heap 1024 1065 464) vs (heap 1024 7561 376)
As a consequence GC triggers way less often and in the total result
count less. This makes a huge difference in a test like this where GC
time contribute to ~50% of the perf result.
It's not easy to start we with a comparable status for the GC in this
situation so I think makes sense to compare just the execution time
having removed the GC form the equation. If we do this:
| | bytecode | native | native | speed 3 vs |
| | | speed 2 | speed 3 | bytecode |
|-------------+----------+---------+---------+------------|
| non GC time | 80 | 49 | 44 | 1.8x |
| GC time | 77 | 46 | 48 | |
| Total | 157 | 95 | 92 | |
So ~1.8x of execution time excluding GC for the native compiler in the
current statu.
Please let me know if something looks wrong.
There's an heap of room for improvements but I think I'll leave the perf
focus for now to continue the bring-up of the missing essential
features.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-01 10:34 ` Andrea Corallo
@ 2019-12-01 15:27 ` Stefan Monnier
0 siblings, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2019-12-01 15:27 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> I've discovered that my previous measure made not much sense because the
> time spent into GC is much bigger during the byte-code runs.
The GC time should never be discarded: it's real work that is being performed.
> My explanation for that (you'll tell me if it makes sense) is that
> having to load by hand all the .eln we start the test native compiled
> tests with a bigger heap:
That's possible, but a smaller heap also makes for a faster GC.
Usually the time to perform one GC is largely proportional to the size
of the heap, and we set the "time between GC" to be itself proportional
to the size of the heap, so that overall the "time spent in GC" should
be *more or less* independent from the heap size.
Another possible source of difference is that the two codes don't behave
the same w.r.t the amount of allocation they perform.
In any case, to the extent that GC time is reduced, I think we should
take it as a nice side-benefit.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-29 15:10 ` Eli Zaretskii
2019-11-29 15:30 ` Stefan Monnier
@ 2019-11-29 16:43 ` Andrea Corallo
2019-11-29 19:26 ` Eli Zaretskii
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-11-29 16:43 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Date: Fri, 29 Nov 2019 14:59:16 +0000
>> Cc: emacs-devel@gnu.org
>>
>> | | byte-compiled | native-compiled | speed-up |
>> |-------------+---------------+-----------------+----------|
>> | interactive | 27s | 19s | 1.4x |
>> | batch | 15s | 8s | 1.9x |
>>
>> To me makes quite sense that in interactive there's less difference
>> cause of the re-display done in C anyway.
>
> Regardless of the speed-up due to using native code, it sounds strange
> to me that byte-compiling interactively takes twice the time it takes
> non-interactively. I have hard time convincing myself that redisplay
> has such a profound effect on byte compilation, since byte-compiling a
> file normally yields one or two lines of output, regardless of the
> size of the file being compiled.
>
> Does the timing change if you delete the window showing the
> compilation log? In that case, redisplay is not involved at all.
>
> Also, was the interactive session started only once, and then used for
> compiling all the files? Or was it restarted anew for each
> compilation? If the latter, then the slow-down is probably due to a
> slower startup of a GUI session.
>
> Thanks.
>
Hi Eli,
yeah sounds strange. I'm very possibly doing something wrong here.
What I'm doing is starting a fresh "emacs -Q" and then measuring the
time of:
(mapc #'byte-compile-file (directory-files "~/emacs/lisp" t "el$"))
Then is one single start of the GUI for all compilations.
I see an intense message activity going on in the mini-buffer so maybe
that's the reason?
Question how could I delete the compilation log without interrupting the
compilation it-self?
Bests
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-29 16:43 ` Andrea Corallo
@ 2019-11-29 19:26 ` Eli Zaretskii
2019-11-29 21:18 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-11-29 19:26 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Fri, 29 Nov 2019 16:43:21 +0000
>
> (mapc #'byte-compile-file (directory-files "~/emacs/lisp" t "el$"))
>
> Then is one single start of the GUI for all compilations.
>
> I see an intense message activity going on in the mini-buffer so maybe
> that's the reason?
>
> Question how could I delete the compilation log without interrupting the
> compilation it-self?
How about binding inhibit-message to a non-nil value, around that mapc
call?
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-29 19:26 ` Eli Zaretskii
@ 2019-11-29 21:18 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-11-29 21:18 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
>> Date: Fri, 29 Nov 2019 16:43:21 +0000
>>
>> (mapc #'byte-compile-file (directory-files "~/emacs/lisp" t "el$"))
>>
>> Then is one single start of the GUI for all compilations.
>>
>> I see an intense message activity going on in the mini-buffer so maybe
>> that's the reason?
>>
>> Question how could I delete the compilation log without interrupting the
>> compilation it-self?
>
> How about binding inhibit-message to a non-nil value, around that mapc
> call?
>
Great, doing that we are straight down into the 15s territory, exactly as
the measured batch. I guess this explains.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 6:54 On elisp running native Andrea Corallo
` (2 preceding siblings ...)
2019-11-28 16:04 ` Stefan Monnier
@ 2019-12-08 20:18 ` Andrea Corallo
2019-12-26 13:05 ` Andrea Corallo
2020-03-15 14:21 ` Spenser Truex
4 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-08 20:18 UTC (permalink / raw)
To: emacs-devel
Hi all, just a quick update on this.
I've got some time to work on gccemacs and I've implemented the support
for docstrings and interactive functions.
All lexically scoped Elisp functions can now be native compiled with no
exceptions.
I'll try to keep the /master branch as the one with the most updated
working state for the project. I'll also give an heads-up here when
I've some progress to be shared.
I guess next is to be able to produce an Emacs image made of native
Elisp functions.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-08 20:18 ` Andrea Corallo
@ 2019-12-26 13:05 ` Andrea Corallo
2019-12-26 17:30 ` Eli Zaretskii
2019-12-26 20:26 ` Stefan Monnier
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-26 13:05 UTC (permalink / raw)
To: emacs-devel
Hi all,
Another update on gccemacs, this time quite substantial.
It brings in support for: garbage collector, image dump and bootstrap
plus some more.
I tried to write down some details about what I've done and why, see
"Update 2" here:
https://akrl.sdf.org/gccemacs.html
The code is quite fresh but I'm using it right now and seems to work.
I'm looking forward for feedback. Also if somebody is willing to test
it he's welcome.
I'd be also very interested in maintainer's opinion.
Bests
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 13:05 ` Andrea Corallo
@ 2019-12-26 17:30 ` Eli Zaretskii
2019-12-26 18:14 ` Andrea Corallo
2019-12-26 20:26 ` Stefan Monnier
1 sibling, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-26 17:30 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Thu, 26 Dec 2019 13:05:33 +0000
>
> I'd be also very interested in maintainer's opinion.
Does that mean me? Or someone else?
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 17:30 ` Eli Zaretskii
@ 2019-12-26 18:14 ` Andrea Corallo
2019-12-26 20:37 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-26 18:14 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Date: Thu, 26 Dec 2019 13:05:33 +0000
>>
>> I'd be also very interested in maintainer's opinion.
>
> Does that mean me? Or someone else?
I meant all maintainers but I'm absolutely interested in your opinion.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 18:14 ` Andrea Corallo
@ 2019-12-26 20:37 ` Eli Zaretskii
2019-12-26 22:10 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-26 20:37 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: emacs-devel@gnu.org
> Date: Thu, 26 Dec 2019 18:14:37 +0000
>
> Eli Zaretskii <eliz@gnu.org> writes:
>
> >> From: Andrea Corallo <akrl@sdf.org>
> >> Date: Thu, 26 Dec 2019 13:05:33 +0000
> >>
> >> I'd be also very interested in maintainer's opinion.
> >
> > Does that mean me? Or someone else?
>
> I meant all maintainers but I'm absolutely interested in your opinion.
(Well, you said "maintainer's", singular. Maybe that was a typo.)
In any case, I'd ask for more specific questions, please. IOW, please
tell what aspects of this would you want me to state my opinions
about. (I'm not sure I have an opinion about what is of interest to
you, but if I do, I promise to share it.)
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 20:37 ` Eli Zaretskii
@ 2019-12-26 22:10 ` Andrea Corallo
2019-12-27 9:12 ` Eli Zaretskii
2019-12-27 15:53 ` Stefan Monnier
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-26 22:10 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: emacs-devel@gnu.org
>> Date: Thu, 26 Dec 2019 18:14:37 +0000
>>
>> Eli Zaretskii <eliz@gnu.org> writes:
>>
>> >> From: Andrea Corallo <akrl@sdf.org>
>> >> Date: Thu, 26 Dec 2019 13:05:33 +0000
>> >>
>> >> I'd be also very interested in maintainer's opinion.
>> >
>> > Does that mean me? Or someone else?
>>
>> I meant all maintainers but I'm absolutely interested in your opinion.
>
> (Well, you said "maintainer's", singular. Maybe that was a typo.)
>
> In any case, I'd ask for more specific questions, please. IOW, please
> tell what aspects of this would you want me to state my opinions
> about. (I'm not sure I have an opinion about what is of interest to
> you, but if I do, I promise to share it.)
What do think about the general idea of having a compiler able to native
compile elisp?
What do you think about the proposed design (Lisp -> LAP -> LIMPLE ->
libgccjit -> elf)?
Do you see any weakness in some part of the design you would be
particularly worried about?
Do you see in this a potential for having it up-stream in the future?
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 22:10 ` Andrea Corallo
@ 2019-12-27 9:12 ` Eli Zaretskii
2019-12-27 10:46 ` Andrea Corallo
` (2 more replies)
2019-12-27 15:53 ` Stefan Monnier
1 sibling, 3 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-27 9:12 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: emacs-devel@gnu.org
> Date: Thu, 26 Dec 2019 22:10:56 +0000
>
> What do think about the general idea of having a compiler able to native
> compile elisp?
I don't feel I know enough about programming language theory to give a
useful technical opinion. I can only try to play a devil's advocate,
and mention a few potential issues with such an approach:
. it will need a compiler (and GCC at that) on the end-user system
. the code it creates must be used on the same system, or on systems
of very similar architecture, so it will be harder to distribute;
e.g., release tarballs will be unable to include such code, like
we do now with *.elc files
. I wonder how you solve the problem with dependency on libgcc and
its likes? this could be another source of reduced portability
> What do you think about the proposed design (Lisp -> LAP -> LIMPLE ->
> libgccjit -> elf)?
Again, I have nothing intelligent to say about the technical aspects
and validity of this approach, only about its practical implications.
This means it's GCC-specific, and moreover requires native support for
loading ELF objects, right? Which means it will probably run well
only on GNU/Linux and maybe the *BSD systems. That's a bummer for me
personally, because my main development machine runs (gasp!)
MS-Windows. IOW, I will never be able to run or debug this feature,
except by remote login to a GNU/Linux system or some other equally
inconvenient method, because neither ELF loading nor libgccjit are
available on my system. (Or maybe you intend to make both happen on
MS-Windows as part of your work? ;-)
> Do you see any weakness in some part of the design you would be
> particularly worried about?
See above.
I also wonder about the debugging. If this feature goes prime time,
at first I imagine we will need to debug problems with generated code
quite a lot. And since Lisp code frequently calls back into C, we'd
need a good way of debugging them both, and being able to relate the
generated code to the original Lisp while at that. If this is not
possible, or hard to accomplish, and will require people to step
through generated code on the machine language level and correlate it
with Lisp in their heads, it would mean only a handful of people will
be able to solve any such problems.
A typical situation is that Emacs segfaults, and the immediate crash
locus is a bunch of machine instructions -- how would one go about
debugging this? (A similar issue exists with debugging faulty
bytecode, but since the byte compiler is very old and mature, this
almost never is of any practical importance.)
> Do you see in this a potential for having it up-stream in the future?
Yes, assuming you (and any others who would like to actively work on
that branch) will have enough energy and motivation to bring this job
to its completion, including reasonable solutions to the debugging
issue, support for all Emacs configurations we care about (a frequent
gotcha is 32-bit builds --with-wide-int; and what about
--enable-check-lisp-object-type? etc.), and then support it during the
initial stages of its use on master, when people will almost certainly
report problems with it that were never heard of before.
I realize that the above sounds negative, and maybe even discouraging,
and I apologize for that. But please bear with me: if I'm allowed to
put on my project management cap, I have some bitter experience with
several experimental features which required a significant amount of
work, but were abandoned before becoming mature, or were merged
although they weren't mature enough, or whose developers disappeared
without a trace, leaving us to cope with the problems reported against
those features without enough knowledge about the code. So I think
for such a radical new feature, which affects how we compile Lisp, we
(at least I personally) would like to avoid such disappointments as
much as practically possible.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-27 9:12 ` Eli Zaretskii
@ 2019-12-27 10:46 ` Andrea Corallo
2019-12-27 14:03 ` Eli Zaretskii
2019-12-28 1:17 ` Richard Stallman
2019-12-28 13:07 ` Dmitry Gutov
2 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-27 10:46 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> . it will need a compiler (and GCC at that) on the end-user system
> . the code it creates must be used on the same system, or on systems
> of very similar architecture, so it will be harder to distribute;
> e.g., release tarballs will be unable to include such code, like
> we do now with *.elc files
Correct eln are not portable (that's the reason why have some performance
advantage) and they must be compiled as we compile C code.
> This means it's GCC-specific, and moreover requires native support for
> loading ELF objects, right? Which means it will probably run well
> only on GNU/Linux and maybe the *BSD systems. That's a bummer for me
> personally, because my main development machine runs (gasp!)
> MS-Windows. IOW, I will never be able to run or debug this feature,
> except by remote login to a GNU/Linux system or some other equally
> inconvenient method, because neither ELF loading nor libgccjit are
> available on my system. (Or maybe you intend to make both happen on
> MS-Windows as part of your work? ;-)
I'm not sure about libgccjit windows support (probably lacking it?) or
how hard would be to put it in place. I don't have a windows machine to
develop on so it's out of my reachable scope sorry.
At least I see libgccjit ships already with a lot of GNU/Linux
distribution.
I guess this would be a dependency on a library for an optional feature,
as we already have others.
> I also wonder about the debugging. If this feature goes prime time,
> at first I imagine we will need to debug problems with generated code
> quite a lot. And since Lisp code frequently calls back into C, we'd
> need a good way of debugging them both, and being able to relate the
> generated code to the original Lisp while at that. If this is not
> possible, or hard to accomplish, and will require people to step
> through generated code on the machine language level and correlate it
> with Lisp in their heads, it would mean only a handful of people will
> be able to solve any such problems.
>
> A typical situation is that Emacs segfaults, and the immediate crash
> locus is a bunch of machine instructions -- how would one go about
> debugging this? (A similar issue exists with debugging faulty
> bytecode, but since the byte compiler is very old and mature, this
> almost never is of any practical importance.)
Generally speaking the native compiler has to be correct exactly as the
byte-interpreter has.
So yes, if it's broken stepping with gdb or some other debugging
technique in the generated code will probably be useful to understand
why. Exaclty as we would step in the byte-interpreter.
> I realize that the above sounds negative, and maybe even discouraging,
> and I apologize for that. But please bear with me: if I'm allowed to
> put on my project management cap, I have some bitter experience with
> several experimental features which required a significant amount of
> work, but were abandoned before becoming mature, or were merged
> although they weren't mature enough, or whose developers disappeared
> without a trace, leaving us to cope with the problems reported against
> those features without enough knowledge about the code. So I think
> for such a radical new feature, which affects how we compile Lisp, we
> (at least I personally) would like to avoid such disappointments as
> much as practically possible.
I understand your concern on this. I guess is up to me to prove the
opposite with steady contribution.
Thanks for the feedback
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-27 10:46 ` Andrea Corallo
@ 2019-12-27 14:03 ` Eli Zaretskii
0 siblings, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-27 14:03 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: emacs-devel@gnu.org
> Date: Fri, 27 Dec 2019 10:46:50 +0000
>
> > This means it's GCC-specific, and moreover requires native support for
> > loading ELF objects, right? Which means it will probably run well
> > only on GNU/Linux and maybe the *BSD systems. That's a bummer for me
> > personally, because my main development machine runs (gasp!)
> > MS-Windows. IOW, I will never be able to run or debug this feature,
> > except by remote login to a GNU/Linux system or some other equally
> > inconvenient method, because neither ELF loading nor libgccjit are
> > available on my system. (Or maybe you intend to make both happen on
> > MS-Windows as part of your work? ;-)
>
> I'm not sure about libgccjit windows support (probably lacking it?) or
> how hard would be to put it in place. I don't have a windows machine to
> develop on so it's out of my reachable scope sorry.
>
> At least I see libgccjit ships already with a lot of GNU/Linux
> distribution.
AFAIK, it is available only on ELF-based systems, perhaps not even on
all of them.
> I understand your concern on this. I guess is up to me to prove the
> opposite with steady contribution.
Thank you for working on this.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-27 9:12 ` Eli Zaretskii
2019-12-27 10:46 ` Andrea Corallo
@ 2019-12-28 1:17 ` Richard Stallman
2019-12-28 7:50 ` Eli Zaretskii
2019-12-28 8:17 ` Andrea Corallo
2019-12-28 13:07 ` Dmitry Gutov
2 siblings, 2 replies; 210+ messages in thread
From: Richard Stallman @ 2019-12-28 1:17 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel, akrl
[[[ To any NSA and FBI agents reading my email: please consider ]]]
[[[ whether defending the US Constitution against all enemies, ]]]
[[[ foreign or domestic, requires you to follow Snowden's example. ]]]
> This means it's GCC-specific, and moreover requires native support for
> loading ELF objects, right? Which means it will probably run well
> only on GNU/Linux and maybe the *BSD systems.
In principle there is nothing wrong with that. The main purpose of
every GNU package is to make the GNU system better. We cooperate with
support for other systems when that is not difficult, as long as
someone chooses to do the work. But we should not reject a useful
feature simply because it can't be (or hasn't been) implemented on
a non-GNU system.
However, I have doubts that compiling Elisp into machine code is worth
the trouble. And I think it is likely to be fragile. The option
exists of writing C code by hand. Compiling the Elisp code would be
quicker and avoid errors, but will that savings be realized enough
times to justify the time used for developing and maintaining the
compiler? I tend to doubt it.
How about compiling Elisp code into C? That should be a lot less work,
and it might be useful in other ways too.
--
Dr Richard Stallman
Founder, Free Software Foundation (https://gnu.org, https://fsf.org)
Internet Hall-of-Famer (https://internethalloffame.org)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 1:17 ` Richard Stallman
@ 2019-12-28 7:50 ` Eli Zaretskii
2019-12-28 8:17 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 7:50 UTC (permalink / raw)
To: rms; +Cc: akrl, emacs-devel
> From: Richard Stallman <rms@gnu.org>
> Date: Fri, 27 Dec 2019 20:17:57 -0500
> Cc: emacs-devel@gnu.org, akrl@sdf.org
>
> Compiling the Elisp code would be
> quicker and avoid errors, but will that savings be realized enough
> times to justify the time used for developing and maintaining the
> compiler? I tend to doubt it.
Andrea published some numbers here:
https://lists.gnu.org/archive/html/emacs-devel/2019-12/msg00018.html
They seem to show speedup by a factor of 1.8 compared to bytecode.
Another, more synthetic set of benchmarks is here:
https://akrl.sdf.org/gccemacs.html#org0d9acac
Those benchmarks show even larger speedups.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 1:17 ` Richard Stallman
2019-12-28 7:50 ` Eli Zaretskii
@ 2019-12-28 8:17 ` Andrea Corallo
2019-12-29 0:08 ` Richard Stallman
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-28 8:17 UTC (permalink / raw)
To: Richard Stallman; +Cc: Eli Zaretskii, emacs-devel
Richard Stallman <rms@gnu.org> writes:
> However, I have doubts that compiling Elisp into machine code is worth
> the trouble. And I think it is likely to be fragile. The option
> exists of writing C code by hand. Compiling the Elisp code would be
> quicker and avoid errors, but will that savings be realized enough
> times to justify the time used for developing and maintaining the
> compiler? I tend to doubt it.
>
> How about compiling Elisp code into C? That should be a lot less work,
> and it might be useful in other ways too.
Hi, to better explain.
Essentially using libgccjit means already describing C programs.
In other words instead of writing strings into a file, functions to
describe the program you want to compile have to be called. Libgccjit
let you describe through these a program with a C-like semantic.
Regardless the programming language is written in, libgccjit API is
exactly what I would re-implement as interface to emit strings to form a
proper C file as output.
This task should be relative straight forward having all the system
already working and the the libgccjit API to copy. But what we would
loose is the already existing libgccjit infrastructure plus we would
have to deal with files and compilation could not happen "in memory"
having to go through the file-system.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 8:17 ` Andrea Corallo
@ 2019-12-29 0:08 ` Richard Stallman
2019-12-29 9:29 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Richard Stallman @ 2019-12-29 0:08 UTC (permalink / raw)
To: Andrea Corallo; +Cc: eliz, emacs-devel
[[[ To any NSA and FBI agents reading my email: please consider ]]]
[[[ whether defending the US Constitution against all enemies, ]]]
[[[ foreign or domestic, requires you to follow Snowden's example. ]]]
> Essentially using libgccjit means already describing C programs.
> In other words instead of writing strings into a file, functions to
> describe the program you want to compile have to be called. Libgccjit
> let you describe through these a program with a C-like semantic.
That means it is in some sense equivalent to generating a C program,
My point is that it might be better to generate an honest-to-goodness
C program, since Emacs already has the facility to load those.
--
Dr Richard Stallman
Founder, Free Software Foundation (https://gnu.org, https://fsf.org)
Internet Hall-of-Famer (https://internethalloffame.org)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-29 0:08 ` Richard Stallman
@ 2019-12-29 9:29 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-29 9:29 UTC (permalink / raw)
To: Richard Stallman; +Cc: eliz, emacs-devel
Richard Stallman <rms@gnu.org> writes:
> That means it is in some sense equivalent to generating a C program,
> My point is that it might be better to generate an honest-to-goodness
> C program, since Emacs already has the facility to load those.
This is another point but I think is quite independent.
In fact would be possible to target the current module interface with
libgccjit as vice versa would be possible to target the "new" ABI with C
(as it's under discussion).
I decided not to target the module interface for performance reasons.
AFAIK module interface was never designed with performance as primary
goal (please correct me if I'm wrong).
It's very important to have a fast interface for this application at the
price of sacrificing over other aspects as stability on different
versions for instance.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-27 9:12 ` Eli Zaretskii
2019-12-27 10:46 ` Andrea Corallo
2019-12-28 1:17 ` Richard Stallman
@ 2019-12-28 13:07 ` Dmitry Gutov
2019-12-28 13:28 ` Eli Zaretskii
` (3 more replies)
2 siblings, 4 replies; 210+ messages in thread
From: Dmitry Gutov @ 2019-12-28 13:07 UTC (permalink / raw)
To: Eli Zaretskii, Andrea Corallo; +Cc: emacs-devel
On 27.12.2019 11:12, Eli Zaretskii wrote:
> IOW, I will never be able to run or debug this feature,
> except by remote login to a GNU/Linux system or some other equally
> inconvenient method, because neither ELF loading nor libgccjit are
> available on my system. (Or maybe you intend to make both happen on
> MS-Windows as part of your work?;-)
Considering WSL is getting more popular and sees some solid development
these days, what are the odds that this is going to be our sole target
on Windows, some years since?
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-28 13:07 ` Dmitry Gutov
@ 2019-12-28 13:28 ` Eli Zaretskii
2019-12-28 19:56 ` Óscar Fuentes
` (2 subsequent siblings)
3 siblings, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 13:28 UTC (permalink / raw)
To: Dmitry Gutov; +Cc: emacs-devel, akrl
> Cc: emacs-devel@gnu.org
> From: Dmitry Gutov <dgutov@yandex.ru>
> Date: Sat, 28 Dec 2019 16:07:21 +0300
>
> Considering WSL is getting more popular and sees some solid development
> these days, what are the odds that this is going to be our sole target
> on Windows, some years since?
I have no idea.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-28 13:07 ` Dmitry Gutov
2019-12-28 13:28 ` Eli Zaretskii
@ 2019-12-28 19:56 ` Óscar Fuentes
2019-12-29 19:59 ` chad
2019-12-30 8:47 ` Achim Gratz
3 siblings, 0 replies; 210+ messages in thread
From: Óscar Fuentes @ 2019-12-28 19:56 UTC (permalink / raw)
To: emacs-devel
Dmitry Gutov <dgutov@yandex.ru> writes:
> Considering WSL is getting more popular and sees some solid
> development these days, what are the odds that this is going to be our
> sole target on Windows, some years since?
Considering that Emacs still supports Windows 98 and WSL is a recent
addition to Windows 10... go figure :-)
Anyways, AFAIK WSL has several limitations on its interaction with
Windows proper and not everybody would happily switch to a POSIX
environment.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 13:07 ` Dmitry Gutov
2019-12-28 13:28 ` Eli Zaretskii
2019-12-28 19:56 ` Óscar Fuentes
@ 2019-12-29 19:59 ` chad
2019-12-30 8:17 ` arthur miller
2019-12-30 13:56 ` Dmitry Gutov
2019-12-30 8:47 ` Achim Gratz
3 siblings, 2 replies; 210+ messages in thread
From: chad @ 2019-12-29 19:59 UTC (permalink / raw)
To: Dmitry Gutov; +Cc: Eli Zaretskii, EMACS development team, Andrea Corallo
[-- Attachment #1: Type: text/plain, Size: 1365 bytes --]
On Sat, Dec 28, 2019 at 5:07 AM Dmitry Gutov <dgutov@yandex.ru> wrote:
> Considering WSL is getting more popular and sees some solid development
> these days, what are the odds that this is going to be our sole target
> on Windows, some years since?
>
Microsoft is moving the WSL 'platform' away from translating POSIX calls
into Windows-native (W32, Win32, I forget the details), and instead
implementing a system where the Linux kernel runs in a custom VM (actually,
inside a container running inside a VM, if I understand it correctly). The
system then provides the necessary software to bridge the two systems from
the user POV. The architecture is supposedly very close to what Google (the
software part, not the Advertising company part) is doing with ChromeOS and
their Crostini project, where the user's choice of distribution runs
(kernel and all) alongside the Linux kernel that underlies ChromeOS, and
software inside each separate OS is used to bridge (for example) X11 into
the Wayland-based display engine, share files between the two, etc.
Supposedly, this was the way they found to solve the performance problems
with the impedance mismatch between Windows and Linux design issues
(especially filesystem operations).
If this works, then it'll make Windows support easier, since it will be
very close to GNU/Linux support.
Hope that helps,
~Chad
[-- Attachment #2: Type: text/html, Size: 1802 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* RE: On elisp running native
2019-12-29 19:59 ` chad
@ 2019-12-30 8:17 ` arthur miller
2019-12-30 14:09 ` Dmitry Gutov
2019-12-30 15:25 ` Eli Zaretskii
2019-12-30 13:56 ` Dmitry Gutov
1 sibling, 2 replies; 210+ messages in thread
From: arthur miller @ 2019-12-30 8:17 UTC (permalink / raw)
To: chad, Dmitry Gutov; +Cc: Eli Zaretskii, Andrea Corallo, EMACS development team
[-- Attachment #1: Type: text/plain, Size: 3080 bytes --]
I haven't yet installed WSL, nor do I have any plans. I don't see why should I install entire Linux kernel if I can run applications natively. I don't understand why people are so excited about wsl when they could just run Linux kernel on bare metal? I do it 99% of time myself and I don't miss Windows at slightest.
Mingw and/or Cygwin do just fine for the apps that does not have native win ports already, but most worth-have apps are already written in portable way. Running entire wm and another kernel just to run an app is an overkill on resources in my eyes.
I would like to see a world where we go towards using less resources instead of just piling platform over platform eating all CPU and ram achievements just because it's easy way out. I think it's madness to run a chat app that requires 1.5 gig or dependency (slack on blink), or as someone on YT recommends to run a virtual machine on Windows with GNU/Linux distro for the sole purpose of running Emacs. WSL is towards that trend. Of course if Emacs run on WSL, and of course it will, is not a problem, but if Emacs does not have native port, that will be a problem.
So I write this just to express that some people value native win32 port and hope native win32 port will not go away or get behind just because it is *easier* to run Emacs on wsl pile of ...
Skickat från min Samsung Galaxy-smartphone.
-------- Originalmeddelande --------
Från: chad <yandros@gmail.com>
Datum: 2019-12-29 21:00 (GMT+01:00)
Till: Dmitry Gutov <dgutov@yandex.ru>
Kopia: Eli Zaretskii <eliz@gnu.org>, EMACS development team <emacs-devel@gnu.org>, Andrea Corallo <akrl@sdf.org>
Ämne: Re: On elisp running native
On Sat, Dec 28, 2019 at 5:07 AM Dmitry Gutov <dgutov@yandex.ru<mailto:dgutov@yandex.ru>> wrote:
Considering WSL is getting more popular and sees some solid development
these days, what are the odds that this is going to be our sole target
on Windows, some years since?
Microsoft is moving the WSL 'platform' away from translating POSIX calls into Windows-native (W32, Win32, I forget the details), and instead implementing a system where the Linux kernel runs in a custom VM (actually, inside a container running inside a VM, if I understand it correctly). The system then provides the necessary software to bridge the two systems from the user POV. The architecture is supposedly very close to what Google (the software part, not the Advertising company part) is doing with ChromeOS and their Crostini project, where the user's choice of distribution runs (kernel and all) alongside the Linux kernel that underlies ChromeOS, and software inside each separate OS is used to bridge (for example) X11 into the Wayland-based display engine, share files between the two, etc. Supposedly, this was the way they found to solve the performance problems with the impedance mismatch between Windows and Linux design issues (especially filesystem operations).
If this works, then it'll make Windows support easier, since it will be very close to GNU/Linux support.
Hope that helps,
~Chad
[-- Attachment #2: Type: text/html, Size: 4237 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-30 8:17 ` arthur miller
@ 2019-12-30 14:09 ` Dmitry Gutov
2019-12-30 15:25 ` Eli Zaretskii
1 sibling, 0 replies; 210+ messages in thread
From: Dmitry Gutov @ 2019-12-30 14:09 UTC (permalink / raw)
To: arthur miller, chad; +Cc: Eli Zaretskii, Andrea Corallo, EMACS development team
On 30.12.2019 10:17, arthur miller wrote:
> I haven't yet installed WSL, nor do I have any plans. I don't see why
> should I install entire Linux kernel if I can run applications natively.
> I don't understand why people are so excited about wsl when they could
> just run Linux kernel on bare metal? I do it 99% of time myself and I
> don't miss Windows at slightest.
So do I. We are not the target audience, are we?
> Mingw and/or Cygwin do just fine for the apps that does not have native
> win ports already, but most worth-have apps are already written in
> portable way. Running entire wm and another kernel just to run an app is
> an overkill on resources in my eyes.
How big is a kernel? 5 MB? 50? It's tiny compared to the amounts of RAM
we have these days.
We either pile on abstractions, or multiply different versions of code
designed to do the same thing.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-30 8:17 ` arthur miller
2019-12-30 14:09 ` Dmitry Gutov
@ 2019-12-30 15:25 ` Eli Zaretskii
2019-12-30 22:03 ` arthur miller
1 sibling, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-30 15:25 UTC (permalink / raw)
To: arthur miller; +Cc: akrl, yandros, emacs-devel, dgutov
> From: arthur miller <arthur.miller@live.com>
> CC: Eli Zaretskii <eliz@gnu.org>, EMACS development team
> <emacs-devel@gnu.org>, Andrea Corallo <akrl@sdf.org>
> Date: Mon, 30 Dec 2019 08:17:24 +0000
>
> Mingw and/or Cygwin do just fine for the apps that does not have native win ports already
FTR: MinGW produces native Windows executables. Maybe you were
thinking about MSYS.
^ permalink raw reply [flat|nested] 210+ messages in thread
* RE: On elisp running native
2019-12-30 15:25 ` Eli Zaretskii
@ 2019-12-30 22:03 ` arthur miller
2019-12-31 3:34 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: arthur miller @ 2019-12-30 22:03 UTC (permalink / raw)
To: Eli Zaretskii
Cc: akrl@sdf.org, yandros@gmail.com, emacs-devel@gnu.org,
dgutov@yandex.ru
[-- Attachment #1: Type: text/plain, Size: 2258 bytes --]
Yes, I know.
With apps that don't have native ports I ment *nix applications that are not written with portability in mind. For example Firefox, Apache or similar are already written to run native, while say binutils or some other typical posix apps, usually need extra portability layer to compile on win32. mingw/cygwin/msys usually provide enough of that portability layer for applications to be "relatively easily" ported over to native win32. In that context I don't really see what gap wsl fills.
I have no idea whom Microsoft targets with wsl, but it is not their first time they add posix layer to Windows. Before wsl there was sfu and before that it was Interix, and before that was probably something else which I don't remember. Neither Interix nor SFU catched big time, I am not sure why would wsl?
Sure, this time they parted with Ubuntu which is big name on GNU/Linux marketing side so maybe this time it leaves bigger ripples behind, No idea, time will tell. Maybe Microsoft just realized that most enterprise applications move to cloud and certain mobile OS:s got enough of traction to become everyday alternative for masses. That maybe leaves Windows as less important as desktop OS in future and they feel a need to target more niche clients such as sys admins and backend developers? I don't know, I don't think this dev list is right place for such speculations so sorry for the regression.
I just wanted to express my concern for not leaving mative win32 port as a secondary citizen in favor of wsl.
Skickat från min Samsung Galaxy-smartphone.
-------- Originalmeddelande --------
Från: Eli Zaretskii <eliz@gnu.org>
Datum: 2019-12-30 16:25 (GMT+01:00)
Till: arthur miller <arthur.miller@live.com>
Kopia: yandros@gmail.com, dgutov@yandex.ru, emacs-devel@gnu.org, akrl@sdf.org
Ämne: Re: On elisp running native
> From: arthur miller <arthur.miller@live.com>
> CC: Eli Zaretskii <eliz@gnu.org>, EMACS development team
> <emacs-devel@gnu.org>, Andrea Corallo <akrl@sdf.org>
> Date: Mon, 30 Dec 2019 08:17:24 +0000
>
> Mingw and/or Cygwin do just fine for the apps that does not have native win ports already
FTR: MinGW produces native Windows executables. Maybe you were
thinking about MSYS.
[-- Attachment #2: Type: text/html, Size: 3394 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-30 22:03 ` arthur miller
@ 2019-12-31 3:34 ` Eli Zaretskii
0 siblings, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-31 3:34 UTC (permalink / raw)
To: arthur miller; +Cc: akrl, yandros, emacs-devel, dgutov
> From: arthur miller <arthur.miller@live.com>
> CC: "yandros@gmail.com" <yandros@gmail.com>, "dgutov@yandex.ru"
> <dgutov@yandex.ru>, "emacs-devel@gnu.org" <emacs-devel@gnu.org>,
> "akrl@sdf.org" <akrl@sdf.org>
> Date: Mon, 30 Dec 2019 22:03:43 +0000
>
> With apps that don't have native ports I ment *nix applications that are not written with portability in mind. For
> example Firefox, Apache or similar are already written to run native, while say binutils or some other typical
> posix apps, usually need extra portability layer to compile on win32. mingw/cygwin/msys usually provide
> enough of that portability layer for applications to be "relatively easily" ported over to native win32. In that
> context I don't really see what gap wsl fills.
I'm not sure we should continue this, as it seems to be wandering
off-topic, but again, FTR, MinGW provides very little to aid porting
applications that weren't written to be portable. A few Posix
headers, a small number of library functions, and that's about it.
The bulk of the job of porting an application to native Windows is on
the shoulders of whoever does the porting. E.g., look at the stuff in
Emacs's w32.c: it all is needed with MinGW.
> I just wanted to express my concern for not leaving mative win32 port as a secondary citizen in favor of wsl.
That depends on contributors to Emacs. If they keep the native
Windows port alive, it will remain.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-29 19:59 ` chad
2019-12-30 8:17 ` arthur miller
@ 2019-12-30 13:56 ` Dmitry Gutov
2019-12-30 14:04 ` Juanma Barranquero
1 sibling, 1 reply; 210+ messages in thread
From: Dmitry Gutov @ 2019-12-30 13:56 UTC (permalink / raw)
To: chad; +Cc: Eli Zaretskii, EMACS development team, Andrea Corallo
On 29.12.2019 21:59, chad wrote:
> Microsoft is moving the WSL 'platform' away from translating POSIX calls
> into Windows-native (W32, Win32, I forget the details), and instead
> implementing a system where the Linux kernel runs in a custom VM
> (actually, inside a container running inside a VM, if I understand it
> correctly).
There are probably solid technical reasons for this, but I think it's
pretty funny that to improve the I/O performance they had to *add* a
layer of abstraction.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-30 13:56 ` Dmitry Gutov
@ 2019-12-30 14:04 ` Juanma Barranquero
0 siblings, 0 replies; 210+ messages in thread
From: Juanma Barranquero @ 2019-12-30 14:04 UTC (permalink / raw)
To: Dmitry Gutov; +Cc: chad, Andrea Corallo, Eli Zaretskii, EMACS development team
[-- Attachment #1: Type: text/plain, Size: 322 bytes --]
On Mon, Dec 30, 2019 at 2:57 PM Dmitry Gutov <dgutov@yandex.ru> wrote:
> There are probably solid technical reasons for this, but I think it's
> pretty funny that to improve the I/O performance they had to *add* a
> layer of abstraction.
Generally speaking, buffering is a layer of abstraction and improves
performance.
[-- Attachment #2: Type: text/html, Size: 434 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 13:07 ` Dmitry Gutov
` (2 preceding siblings ...)
2019-12-29 19:59 ` chad
@ 2019-12-30 8:47 ` Achim Gratz
3 siblings, 0 replies; 210+ messages in thread
From: Achim Gratz @ 2019-12-30 8:47 UTC (permalink / raw)
To: emacs-devel
Am 28.12.2019 um 14:07 schrieb Dmitry Gutov:
> Considering WSL is getting more popular and sees some solid development
> these days, what are the odds that this is going to be our sole target
> on Windows, some years since?
Zero if you want to have proper integration of Emacs into the Windows
(desktop and network) environment. The stated goal for WSL is
development of cloud applications, so it is too isolated from Windows
and WSL2 is making that isolation even stronger (by running a proper
paravirtualized Linux kernel in a somewhat de-featured HyperV VM).
--
Achim.
(on the road :-)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 22:10 ` Andrea Corallo
2019-12-27 9:12 ` Eli Zaretskii
@ 2019-12-27 15:53 ` Stefan Monnier
2019-12-27 21:02 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2019-12-27 15:53 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> What do think about the general idea of having a compiler able to native
> compile elisp?
>
> What do you think about the proposed design (Lisp -> LAP -> LIMPLE ->
> libgccjit -> elf)?
I think the general design is quite sound, yes.
> Do you see any weakness in some part of the design you would be
> particularly worried about?
- Beside the issue of portability/availability of libgccjit, there's
also the issue of stability of libgccjit. More specifically, what is
the probability that libgccjit will still exist more or less in the
same form in 10 or 20 years? So far GCC's main entry point has been
the command line, so we can expect that running `gcc` will still
work 20 years from now. Similarly, LLVM's main entry point is
an API that builds a kind of CFG representation of code, so we can
expect that to still exist 20 years from now. But to the extent that
libgccjit is not very widely used, there's a risk it will completely
disappear or change radically, especially since we use it here in
a way contrary to what its name ("...jit") suggests.
- Going through the byte-compiler's LAP code is convenient but not
ideal. Eventually, you'll want to circumvent bytecomp.el and write
fresh new code to go from Lisp to LIMPLE without the detour through
a stack machine (and probably not in a single step either).
It's not a source of *worry*, tho.
> Do you see in this a potential for having it up-stream in the future?
Yes, tho it will take time.
The main benefit of such a compiler is not to run existing Elisp code
faster (99% of existing Elisp code runs fast enough that the user won't
notice if it runs faster) but to make it practical to write *other*
Elisp code which would otherwise be too slow. But for that to work
well, you want the new compiler to be available "everywhere", rather
than just on some platforms. So it will only start being useful when it
works on GNU/Linux, macOS, Windows, ARM, RISC-V, x86, amd64, MIPS,
younameit (AFAIK lbgccjit's CPU coverage is already good enough for
that, so the main barrier here is the OS support), and when it's not
just an option at compile-time but when it's included in all builds.
So, first it will take time for the compiler to work in all these cases,
then for the feature to be enabled by default, and then for code to
start making actual use of it.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-27 15:53 ` Stefan Monnier
@ 2019-12-27 21:02 ` Andrea Corallo
2019-12-27 22:19 ` Stefan Monnier
2019-12-28 8:34 ` Eli Zaretskii
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-27 21:02 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> - Beside the issue of portability/availability of libgccjit, there's
> also the issue of stability of libgccjit. More specifically, what is
> the probability that libgccjit will still exist more or less in the
> same form in 10 or 20 years? So far GCC's main entry point has been
> the command line, so we can expect that running `gcc` will still
> work 20 years from now. Similarly, LLVM's main entry point is
> an API that builds a kind of CFG representation of code, so we can
> expect that to still exist 20 years from now. But to the extent that
> libgccjit is not very widely used, there's a risk it will completely
> disappear or change radically, especially since we use it here in
> a way contrary to what its name ("...jit") suggests.
Libgccjit is modeled following a C-like semantic, I guess because plugs
where a conventional Gcc front-end would. Consequently I've no reason
to think this should change but yes... this is not a strong proof I get
it.
Probably this is more relevant:
https://gcc.gnu.org/onlinedocs/jit/topics/compatibility.html
When a new entry point is added this is always just and additive change
to the API. At least this is the policy I understand and follow if I
work into it.
> a way contrary to what its name ("...jit") suggests.
That's an interesting point. I think the only two differences from the
"typical" jit use to the one we do are:
- We never leak pointers as immediate into the generated code.
- We do not ask libgccjit to perform the library load cause we do it
with our machinery.
And this let me think to Eli's point on windows missing compatibility.
There's good chance that the reason for this is that libgccjit does use
dlopen&friends. Actually these are exactly the features of libgccjit
infrastructure we do *not* use.
I've decided to use libgccjit cause I thought was the nicest technical
solution. (admittedly quite GNUish :)
One could potentially write another final-pass targeting C from LIMPLE.
Thanks for the feedback.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-27 21:02 ` Andrea Corallo
@ 2019-12-27 22:19 ` Stefan Monnier
2019-12-28 8:34 ` Eli Zaretskii
1 sibling, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2019-12-27 22:19 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> Libgccjit is modeled following a C-like semantic, I guess because plugs
> where a conventional Gcc front-end would. Consequently I've no reason
> to think this should change but yes... this is not a strong proof I get
> it.
I do expect it to change over time, but that's not what I'm worried
about. The worry is more the fact that since it's not very widely used
AFAIK there's a higher risk it could be completely dropped at some
point, or superseded by a another API that looks quite different,
e.g. because it plugs into a different stage of the compiler.
None of this is really foreseeable and I'd expect that it wouldn't be
too hard to retarget LIMPLE to another backend (Emacs is pretty popular
within the programming-language research community, so there's a fairly
good chance we could find people able and willing to take on the
challenge if/when it comes up, so I'm confident that it's not too
serious a worry), but it's still part of the things a maintainer has to
worry about.
> And this let me think to Eli's point on windows missing compatibility.
> There's good chance that the reason for this is that libgccjit does use
> dlopen&friends. Actually these are exactly the features of libgccjit
> infrastructure we do *not* use.
> One could potentially write another final-pass targeting C from LIMPLE.
Generating C code sounds painful to me compared to using libgccjit.
It must be possible to solve the Windows compatibility in a more
direct way.
> I've decided to use libgccjit cause I thought was the nicest technical
> solution. (admittedly quite GNUish :)
I think it was a sound choice.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-27 21:02 ` Andrea Corallo
2019-12-27 22:19 ` Stefan Monnier
@ 2019-12-28 8:34 ` Eli Zaretskii
2019-12-28 8:56 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 8:34 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: Eli Zaretskii <eliz@gnu.org>, emacs-devel@gnu.org
> Date: Fri, 27 Dec 2019 21:02:50 +0000
>
> And this let me think to Eli's point on windows missing compatibility.
> There's good chance that the reason for this is that libgccjit does use
> dlopen&friends. Actually these are exactly the features of libgccjit
> infrastructure we do *not* use.
I'm not sure I understood you, but dlopen, dlsym etc. (or their moral
equivalents) are readily available on Windows; see src/dynlib.c. So
that cannot be the reason why libgccjit is not available on Windows.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 8:34 ` Eli Zaretskii
@ 2019-12-28 8:56 ` Andrea Corallo
2019-12-28 10:06 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-28 8:56 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: Eli Zaretskii <eliz@gnu.org>, emacs-devel@gnu.org
>> Date: Fri, 27 Dec 2019 21:02:50 +0000
>>
>> And this let me think to Eli's point on windows missing compatibility.
>> There's good chance that the reason for this is that libgccjit does use
>> dlopen&friends. Actually these are exactly the features of libgccjit
>> infrastructure we do *not* use.
>
> I'm not sure I understood you, but dlopen, dlsym etc. (or their moral
> equivalents) are readily available on Windows; see src/dynlib.c. So
> that cannot be the reason why libgccjit is not available on Windows.
Sure, but libgccjit AFAIU just calls directly dlopen
(gcc/gcc/jit/jit-playback.c:2650).
I've just found this interesting old thread:
https://gcc.gnu.org/ml/jit/2015-q3/msg00124.html
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 8:56 ` Andrea Corallo
@ 2019-12-28 10:06 ` Eli Zaretskii
2019-12-28 11:57 ` Andrea Corallo
2020-02-24 23:21 ` Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 10:06 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Sat, 28 Dec 2019 08:56:17 +0000
>
> Eli Zaretskii <eliz@gnu.org> writes:
>
> > I'm not sure I understood you, but dlopen, dlsym etc. (or their moral
> > equivalents) are readily available on Windows; see src/dynlib.c. So
> > that cannot be the reason why libgccjit is not available on Windows.
>
> Sure, but libgccjit AFAIU just calls directly dlopen
> (gcc/gcc/jit/jit-playback.c:2650).
That's OK. One of the MinGW flavors actually provides these functions
directly; for the other it should be easy to write them so that
libgccjit can call them.
> I've just found this interesting old thread:
>
> https://gcc.gnu.org/ml/jit/2015-q3/msg00124.html
Thanks. Not sure what to make from that, especially since AFAIK the
Windows code is PIC by default. The problems they are talking about
are very easy to solve, but somehow no one has yet solved them. Which
might mean there's more to it than meets the eye. (However, I'm
talking out of sheer ignorance, so perhaps you should ask on the GCC
list whether there are any fundamental problems with providing
libgccjit on Windows.)
Maybe you should explain in more detail how will the compiled code be
loaded into Emacs. Is each .el file compiled into a separate shared
library? If so, when the shared library is loaded, what entry point
is called, and how does Emacs know which entry point to call when?
Also, compiling needs gas and ld, right? IOW, people who'd like to
use this feature for Lisp that is not part of Emacs will have to have
GCC and GNU Binutils installed, something that AFAIU is not guaranteed
even on GNU/Linux these days.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 10:06 ` Eli Zaretskii
@ 2019-12-28 11:57 ` Andrea Corallo
2019-12-28 13:34 ` Eli Zaretskii
2020-02-24 23:21 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-28 11:57 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> perhaps you should ask on the GCC
> list whether there are any fundamental problems with providing
> libgccjit on Windows.)
https://gcc.gnu.org/ml/jit/2019-q4/msg00007.html
> Maybe you should explain in more detail how will the compiled code be
> loaded into Emacs. Is each .el file compiled into a separate shared
> library? If so, when the shared library is loaded, what entry point
> is called, and how does Emacs know which entry point to call when?
Every .el file is compiled into a .eln. This is technically a shared
library.
Into the .eln Emacs is expecting to find a bunch of symbols. Two of
these are used for relocating objects and Emacs primitive functions. So
Emacs essentially writes some pointers into them.
Then there's 'top_level_run'. This is a real function that is responsible for
modifying the environment as the execution of the various top level forms
on the original .el file would do.
Emacs jumps into 'top_level_run' and this will call back into Emacs for
evaluating forms and/or defining native functions (effectively new
subrs). This function is obviously not called when resuming from an
image dump.
> Also, compiling needs gas and ld, right? IOW, people who'd like to
> use this feature for Lisp that is not part of Emacs will have to have
> GCC and GNU Binutils installed
Correct, IOW these are libgccjit dependencies.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 11:57 ` Andrea Corallo
@ 2019-12-28 13:34 ` Eli Zaretskii
2019-12-28 14:35 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 13:34 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Sat, 28 Dec 2019 11:57:36 +0000
>
> Every .el file is compiled into a .eln. This is technically a shared
> library.
>
> Into the .eln Emacs is expecting to find a bunch of symbols. Two of
> these are used for relocating objects and Emacs primitive functions. So
> Emacs essentially writes some pointers into them.
Can you tell more about this part? What does this relocation include,
and to what extent the details depend on the .eln file being in ELF
format?
> Then there's 'top_level_run'. This is a real function that is responsible for
> modifying the environment as the execution of the various top level forms
> on the original .el file would do.
>
> Emacs jumps into 'top_level_run' and this will call back into Emacs for
> evaluating forms and/or defining native functions (effectively new
> subrs). This function is obviously not called when resuming from an
> image dump.
So when Emacs calls a Lisp function foo-bar, that call will produce a
call to 'top_level_run' function of the corresponding .eln file, and
then 'top_level_run' of that .eln will call the code produced from
'foo-bar'? How does Emacs know which .eln "provides" foo-bar?
And how would the code in the .eln know to call back into Emacs? When
you compile the native code, do you leave the symbols of Emacs
functions unresolved, and rely on the dynamic linker to resolve them?
If so, this doesn't work on Windows: you will get link errors.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 13:34 ` Eli Zaretskii
@ 2019-12-28 14:35 ` Andrea Corallo
2019-12-28 18:38 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-28 14:35 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
>> Date: Sat, 28 Dec 2019 11:57:36 +0000
>>
>> Every .el file is compiled into a .eln. This is technically a shared
>> library.
>>
>> Into the .eln Emacs is expecting to find a bunch of symbols. Two of
>> these are used for relocating objects and Emacs primitive functions. So
>> Emacs essentially writes some pointers into them.
>
> Can you tell more about this part? What does this relocation include,
> and to what extent the details depend on the .eln file being in ELF
> format?
Sure. About these two:
- A pointer has to be set in the shared library to point to the array
containing all the function pointers of Emacs C primitives and
functions used to support the run-time. Lets call it 'function link
table'.
There is only one copy of this structure and is malloced by Emacs.
This is used by the code into the .eln file in order to jump into
Emacs. All elns share the same 'function link table'.
- The other structure is an array of lisp objects that are the objects
coming from the 'constant vectors' of every byte-compiled function.
This is allocated as static in every eln and initialized during the
eln load process.
To recreate the objects Emacs read a string from the eln it-self and
run the reader. When the objects are revived into memory these are
set into the array one by one.
I do not depend on ELF. I'm already using the functions wrapped by our
'dynlib' in order to execute this machinery.
>> Then there's 'top_level_run'. This is a real function that is responsible for
>> modifying the environment as the execution of the various top level forms
>> on the original .el file would do.
>>
>> Emacs jumps into 'top_level_run' and this will call back into Emacs for
>> evaluating forms and/or defining native functions (effectively new
>> subrs). This function is obviously not called when resuming from an
>> image dump.
>
> So when Emacs calls a Lisp function foo-bar, that call will produce a
> call to 'top_level_run' function of the corresponding .eln file, and
> then 'top_level_run' of that .eln will call the code produced from
> 'foo-bar'? How does Emacs know which .eln "provides" foo-bar?
Sorry I wasn't clear.
'top_level_run' is run once during file load. 'top_level_run' calls
back into Emacs to register 'foo-bar' (as a lightly special subr).
So after that 'foo-bar' is called as every other subr is.
> And how would the code in the .eln know to call back into Emacs? When
See above
> you compile the native code, do you leave the symbols of Emacs
> functions unresolved, and rely on the dynamic linker to resolve them?
> If so, this doesn't work on Windows: you will get link errors.
I just use dynlib_sym to obtain the addresses of these few symbols. The
dynamic linker does not come into play and the rest is handled by the
described mechanism. The fact that the list of primitives expected by
the eln matches the one provided by the system is checked comparing two
hashes duiring the load process. If the eln is not compatible an error
is raised.
> Thanks.
I guess it's easier to read in code than to explain (is really not
much). In case you can find 'load_comp_unit' into:
https://gitlab.com/koral/gccemacs/raw/dev/src/comp.c
Please let me know if I can better clarify some of these points.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 14:35 ` Andrea Corallo
@ 2019-12-28 18:38 ` Eli Zaretskii
2019-12-28 22:13 ` Andrea Corallo
2019-12-31 9:55 ` Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-28 18:38 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Sat, 28 Dec 2019 14:35:46 +0000
>
> I guess it's easier to read in code than to explain (is really not
> much). In case you can find 'load_comp_unit' into:
>
> https://gitlab.com/koral/gccemacs/raw/dev/src/comp.c
Well, it's a 34K-line file full of calls to gcc_jit functions. It's
hard to figure out what it does unless one is already familiar with
libgccjit.
Can you describe what happens using a simple example? Suppose I have
a simple function whose body does
(let ((i 0))
(while (< i something)
(foobar i)
(setq i (1+ i))))
When this is compiled with libgccjit using your code, will the
resulting native code include calls to Fwhile, Fsetq, and Fadd1? Or
will it convert the above to the equivalent machine-language loop? If
the latter, how does it know what to produce for Fwhile, for example?
P.S. Pointers to your code where it does something that answers the
above questions are very fine.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-28 18:38 ` Eli Zaretskii
@ 2019-12-28 22:13 ` Andrea Corallo
2019-12-31 9:55 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-28 22:13 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
>> Date: Sat, 28 Dec 2019 14:35:46 +0000
>>
>> I guess it's easier to read in code than to explain (is really not
>> much). In case you can find 'load_comp_unit' into:
>>
>> https://gitlab.com/koral/gccemacs/raw/dev/src/comp.c
>
> Well, it's a 34K-line file full of calls to gcc_jit functions. It's
> hard to figure out what it does unless one is already familiar with
> libgccjit.
Sure I was just talking about the load process.
> Can you describe what happens using a simple example? Suppose I have
> a simple function whose body does
>
> (let ((i 0))
> (while (< i something)
> (foobar i)
> (setq i (1+ i))))
>
> When this is compiled with libgccjit using your code, will the
> resulting native code include calls to Fwhile, Fsetq, and Fadd1? Or
> will it convert the above to the equivalent machine-language loop? If
> the latter, how does it know what to produce for Fwhile, for example?
>
> P.S. Pointers to your code where it does something that answers the
> above questions are very fine.
>
> Thanks.
I think would be helpful to try writing something more than just a
single mail. I'll try to come-up with something written on the web page
describing the compilation process for your example and post the update
here to discuss.
I'll try to do it in the next days (I apologize I'm a little task
saturated).
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 18:38 ` Eli Zaretskii
2019-12-28 22:13 ` Andrea Corallo
@ 2019-12-31 9:55 ` Andrea Corallo
2019-12-31 15:49 ` Eli Zaretskii
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-31 9:55 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> Can you describe what happens using a simple example? Suppose I have
> a simple function whose body does
>
> (let ((i 0))
> (while (< i something)
> (foobar i)
> (setq i (1+ i))))
>
> When this is compiled with libgccjit using your code, will the
> resulting native code include calls to Fwhile, Fsetq, and Fadd1? Or
> will it convert the above to the equivalent machine-language loop? If
> the latter, how does it know what to produce for Fwhile, for example?
>
> P.S. Pointers to your code where it does something that answers the
> above questions are very fine.
>
> Thanks.
Hi,
I've extended the usual page (https://akrl.sdf.org/gccemacs.html) under
"Update 3" to follow an example of compilation process and load
mechanism as you have suggested.
Hope it helps. Please let me know if something needs more
clarification.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-28 10:06 ` Eli Zaretskii
2019-12-28 11:57 ` Andrea Corallo
@ 2020-02-24 23:21 ` Andrea Corallo
2020-02-25 6:14 ` yyoncho
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-24 23:21 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
>> Date: Sat, 28 Dec 2019 08:56:17 +0000
>>
>> Eli Zaretskii <eliz@gnu.org> writes:
>>
>> > I'm not sure I understood you, but dlopen, dlsym etc. (or their moral
>> > equivalents) are readily available on Windows; see src/dynlib.c. So
>> > that cannot be the reason why libgccjit is not available on Windows.
>>
>> Sure, but libgccjit AFAIU just calls directly dlopen
>> (gcc/gcc/jit/jit-playback.c:2650).
>
> That's OK. One of the MinGW flavors actually provides these functions
> directly; for the other it should be easy to write them so that
> libgccjit can call them.
>
>> I've just found this interesting old thread:
>>
>> https://gcc.gnu.org/ml/jit/2015-q3/msg00124.html
>
> Thanks. Not sure what to make from that, especially since AFAIK the
> Windows code is PIC by default. The problems they are talking about
> are very easy to solve, but somehow no one has yet solved them. Which
> might mean there's more to it than meets the eye. (However, I'm
> talking out of sheer ignorance, so perhaps you should ask on the GCC
> list whether there are any fundamental problems with providing
> libgccjit on Windows.)
I just bumped in this Chinese blog where the guy apparently is running
the native-comp branch on Windows 10 using mingw64:
http://www.albertzhou.net/blog/2020/01/emacs-native-comp.html
To a quick look in libgccjit he had to modify the dlopen&friends part
and added some handling for temporary files. In Emacs comp.c code he
touched some setjmp code.
I know zero about Windows but this should help answering the question if
this port is possible and what's roughly the effort.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-24 23:21 ` Andrea Corallo
@ 2020-02-25 6:14 ` yyoncho
2020-02-25 7:29 ` Eli Zaretskii
2020-04-26 12:00 ` Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: yyoncho @ 2020-02-25 6:14 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
[-- Attachment #1: Type: text/plain, Size: 3293 bytes --]
Hi,
I did benchmarks of json parsing here it is what I get:
| test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) |
tot avg err (s) |
|--------------+----------------+------------+---------+-------------+-----------------|
| json-parsing | 2.57 | 3.32 | 318 | 5.89 |
0.10 |
|--------------+----------------+------------+---------+-------------+-----------------|
| total | 2.57 | 3.32 | 318 | 5.89 |
0.10 |
Native:
| test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) |
tot avg err (s) |
|--------------+----------------+------------+---------+-------------+-----------------|
| json-parsing | 1.71 | 4.31 | 343 | 6.02 |
0.01 |
|--------------+----------------+------------+---------+-------------+-----------------|
| total | 1.71 | 4.31 | 343 | 6.02 |
0.01 |
It looks like this is bellow the average speedup you might wanna take a
look:
Here it is the benchmark code:
(require 'cl-lib)
(require 'json)
(defvar json-parsing-input (with-temp-buffer
(insert-file-contents-literally "benchmarks/sample.json")
(buffer-string)))
(defun elb-json-parsing-entry ()
(cl-loop repeat 1000
do (json-read-from-string json-parsing-input)))
Thanks,
Ivan
On Tue, Feb 25, 2020 at 1:21 AM Andrea Corallo <akrl@sdf.org> wrote:
> Eli Zaretskii <eliz@gnu.org> writes:
>
> >> From: Andrea Corallo <akrl@sdf.org>
> >> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> >> Date: Sat, 28 Dec 2019 08:56:17 +0000
> >>
> >> Eli Zaretskii <eliz@gnu.org> writes:
> >>
> >> > I'm not sure I understood you, but dlopen, dlsym etc. (or their moral
> >> > equivalents) are readily available on Windows; see src/dynlib.c. So
> >> > that cannot be the reason why libgccjit is not available on Windows.
> >>
> >> Sure, but libgccjit AFAIU just calls directly dlopen
> >> (gcc/gcc/jit/jit-playback.c:2650).
> >
> > That's OK. One of the MinGW flavors actually provides these functions
> > directly; for the other it should be easy to write them so that
> > libgccjit can call them.
> >
> >> I've just found this interesting old thread:
> >>
> >> https://gcc.gnu.org/ml/jit/2015-q3/msg00124.html
> >
> > Thanks. Not sure what to make from that, especially since AFAIK the
> > Windows code is PIC by default. The problems they are talking about
> > are very easy to solve, but somehow no one has yet solved them. Which
> > might mean there's more to it than meets the eye. (However, I'm
> > talking out of sheer ignorance, so perhaps you should ask on the GCC
> > list whether there are any fundamental problems with providing
> > libgccjit on Windows.)
>
> I just bumped in this Chinese blog where the guy apparently is running
> the native-comp branch on Windows 10 using mingw64:
>
> http://www.albertzhou.net/blog/2020/01/emacs-native-comp.html
>
> To a quick look in libgccjit he had to modify the dlopen&friends part
> and added some handling for temporary files. In Emacs comp.c code he
> touched some setjmp code.
>
> I know zero about Windows but this should help answering the question if
> this port is possible and what's roughly the effort.
>
> Andrea
>
> --
> akrl@sdf.org
>
>
[-- Attachment #2: Type: text/html, Size: 4819 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-02-25 6:14 ` yyoncho
@ 2020-02-25 7:29 ` Eli Zaretskii
2020-02-25 12:59 ` Andrea Corallo
2020-04-26 12:00 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-02-25 7:29 UTC (permalink / raw)
To: yyoncho; +Cc: emacs-devel, monnier, akrl
> From: yyoncho <yyoncho@gmail.com>
> Date: Tue, 25 Feb 2020 08:14:02 +0200
> Cc: Eli Zaretskii <eliz@gnu.org>, Stefan Monnier <monnier@iro.umontreal.ca>,
> emacs-devel <emacs-devel@gnu.org>
>
> I did benchmarks of json parsing here it is what I get:
>
> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) | tot avg err (s) |
> |--------------+----------------+------------+---------+-------------+-----------------|
> | json-parsing | 2.57 | 3.32 | 318 | 5.89 | 0.10 |
> |--------------+----------------+------------+---------+-------------+-----------------|
> | total | 2.57 | 3.32 | 318 | 5.89 | 0.10 |
>
> Native:
>
> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) | tot avg err (s) |
> |--------------+----------------+------------+---------+-------------+-----------------|
> | json-parsing | 1.71 | 4.31 | 343 | 6.02 | 0.01 |
> |--------------+----------------+------------+---------+-------------+-----------------|
> | total | 1.71 | 4.31 | 343 | 6.02 | 0.01 |
>
> It looks like this is bellow the average speedup you might wanna take a look:
I think this might be expected, since most of the heavy processing in
this benchmark is in C, and thus not affected by the compiling Lisp
into native machine code.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-25 7:29 ` Eli Zaretskii
@ 2020-02-25 12:59 ` Andrea Corallo
2020-02-28 4:34 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-25 12:59 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel, yyoncho, monnier
Eli Zaretskii <eliz@gnu.org> writes:
>> From: yyoncho <yyoncho@gmail.com>
>> Date: Tue, 25 Feb 2020 08:14:02 +0200
>> Cc: Eli Zaretskii <eliz@gnu.org>, Stefan Monnier <monnier@iro.umontreal.ca>,
>> emacs-devel <emacs-devel@gnu.org>
>>
>> I did benchmarks of json parsing here it is what I get:
>>
>> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) | tot avg err (s) |
>> |--------------+----------------+------------+---------+-------------+-----------------|
>> | json-parsing | 2.57 | 3.32 | 318 | 5.89 | 0.10 |
>> |--------------+----------------+------------+---------+-------------+-----------------|
>> | total | 2.57 | 3.32 | 318 | 5.89 | 0.10 |
>>
>> Native:
>>
>> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg (s) | tot avg err (s) |
>> |--------------+----------------+------------+---------+-------------+-----------------|
>> | json-parsing | 1.71 | 4.31 | 343 | 6.02 | 0.01 |
>> |--------------+----------------+------------+---------+-------------+-----------------|
>> | total | 1.71 | 4.31 | 343 | 6.02 | 0.01 |
>>
>> It looks like this is bellow the average speedup you might wanna take a look:
>
> I think this might be expected, since most of the heavy processing in
> this benchmark is in C, and thus not affected by the compiling Lisp
> into native machine code.
Yeah, the interesting part would be the GC one. I'm wondering if that's
just a noise artifact or not.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-25 12:59 ` Andrea Corallo
@ 2020-02-28 4:34 ` Eli Zaretskii
2020-02-28 8:26 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-02-28 4:34 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel, yyoncho, monnier
> From: Andrea Corallo <akrl@sdf.org>
> Cc: yyoncho <yyoncho@gmail.com>, monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Tue, 25 Feb 2020 12:59:56 +0000
>
> > I think this might be expected, since most of the heavy processing in
> > this benchmark is in C, and thus not affected by the compiling Lisp
> > into native machine code.
>
> Yeah, the interesting part would be the GC one. I'm wondering if that's
> just a noise artifact or not.
I'm not sure I understand what is interesting about it. JSON parsing
is implemented in C, but it creates Lisp objects (a lot of them), and
thus GC during that processing should not be unexpected.
Of course, if you want to make sure the GC in this case indeed only
reclaims Lisp objects produced by JSON parsing, and is not related to
the native code itself, that's a good thing to check.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-28 4:34 ` Eli Zaretskii
@ 2020-02-28 8:26 ` Andrea Corallo
2020-02-28 9:23 ` yyoncho
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-28 8:26 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel, yyoncho, monnier
Eli Zaretskii <eliz@gnu.org> writes:
> Of course, if you want to make sure the GC in this case indeed only
> reclaims Lisp objects produced by JSON parsing, and is not related to
> the native code itself, that's a good thing to check.
Yes, yyoncho is observing an higher GC time for the native compiled test
and there should be no reason. Maybe is something real maybe not. I
think is worth having a look.
I'll do it in the following day. PS yyoncho if you still have that json
or a similar one that would help.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-28 8:26 ` Andrea Corallo
@ 2020-02-28 9:23 ` yyoncho
0 siblings, 0 replies; 210+ messages in thread
From: yyoncho @ 2020-02-28 9:23 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
[-- Attachment #1: Type: text/plain, Size: 785 bytes --]
I was using random package.json file like this one:
https://github.com/redhat-developer/vscode-java/blob/master/package.json
Thanks,
Ivan
On Fri, Feb 28, 2020 at 10:26 AM Andrea Corallo <akrl@sdf.org> wrote:
> Eli Zaretskii <eliz@gnu.org> writes:
>
> > Of course, if you want to make sure the GC in this case indeed only
> > reclaims Lisp objects produced by JSON parsing, and is not related to
> > the native code itself, that's a good thing to check.
>
> Yes, yyoncho is observing an higher GC time for the native compiled test
> and there should be no reason. Maybe is something real maybe not. I
> think is worth having a look.
>
> I'll do it in the following day. PS yyoncho if you still have that json
> or a similar one that would help.
>
> Andrea
>
> --
> akrl@sdf.org
>
[-- Attachment #2: Type: text/html, Size: 1371 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-25 6:14 ` yyoncho
2020-02-25 7:29 ` Eli Zaretskii
@ 2020-04-26 12:00 ` Andrea Corallo
2020-04-27 4:41 ` yyoncho
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-04-26 12:00 UTC (permalink / raw)
To: yyoncho; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
yyoncho <yyoncho@gmail.com> writes:
> Hi,
>
> I did benchmarks of json parsing here it is what I get:
>
> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg
> (s) | tot avg err (s) |
> |
> --------------+----------------+------------+---------+-------------+-----------------
> |
> | json-parsing | 2.57 | 3.32 | 318 |
> 5.89 | 0.10 |
> |
> --------------+----------------+------------+---------+-------------+-----------------
> |
> | total | 2.57 | 3.32 | 318 |
> 5.89 | 0.10 |
>
>
> Native:
>
> | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg
> (s) | tot avg err (s) |
> |
> --------------+----------------+------------+---------+-------------+-----------------
> |
> | json-parsing | 1.71 | 4.31 | 343 |
> 6.02 | 0.01 |
> |
> --------------+----------------+------------+---------+-------------+-----------------
> |
> | total | 1.71 | 4.31 | 343 |
> 6.02 | 0.01 |
>
> It looks like this is bellow the average speedup you might wanna take
> a look:
>
> Here it is the benchmark code:
>
> (require 'cl-lib)
> (require 'json)
>
> (defvar json-parsing-input (with-temp-buffer
> (insert-file-contents-literally "benchmarks/sample.json")
> (buffer-string)))
>
>
> (defun elb-json-parsing-entry ()
> (cl-loop repeat 1000
> do (json-read-from-string json-parsing-input)))
>
> Thanks,
> Ivan
Hi Ivan,
I've got time to re-look into this.
The perf difference was due to GC time (as Eli pointed out all json
computation is in the C world).
I believe this delta was due to the greater number of objects allocated
in the native comp variant with the consequent more time needed to
traverse them during GC.
Since these measures I've made several changes to reduce the number of
unnecessary live objects. Repeating the test on the current native-comp
against master I see on my machine a performance delta of 2% in favor of
master. Considering I expect this effect to fade out on a real session
with more allocated objects I consider this being acceptable.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-04-26 12:00 ` Andrea Corallo
@ 2020-04-27 4:41 ` yyoncho
0 siblings, 0 replies; 210+ messages in thread
From: yyoncho @ 2020-04-27 4:41 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
[-- Attachment #1: Type: text/plain, Size: 2487 bytes --]
I am glad that my report helped to improve native-comp branch.
Thanks,
Ivan
On Sun, Apr 26, 2020 at 3:00 PM Andrea Corallo <akrl@sdf.org> wrote:
> yyoncho <yyoncho@gmail.com> writes:
>
> > Hi,
> >
> > I did benchmarks of json parsing here it is what I get:
> >
> > | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg
> > (s) | tot avg err (s) |
> > |
> >
> --------------+----------------+------------+---------+-------------+-----------------
> > |
> > | json-parsing | 2.57 | 3.32 | 318 |
> > 5.89 | 0.10 |
> > |
> >
> --------------+----------------+------------+---------+-------------+-----------------
> > |
> > | total | 2.57 | 3.32 | 318 |
> > 5.89 | 0.10 |
> >
> >
> > Native:
> >
> > | test | non-gc avg (s) | gc avg (s) | gcs avg | tot avg
> > (s) | tot avg err (s) |
> > |
> >
> --------------+----------------+------------+---------+-------------+-----------------
> > |
> > | json-parsing | 1.71 | 4.31 | 343 |
> > 6.02 | 0.01 |
> > |
> >
> --------------+----------------+------------+---------+-------------+-----------------
> > |
> > | total | 1.71 | 4.31 | 343 |
> > 6.02 | 0.01 |
> >
> > It looks like this is bellow the average speedup you might wanna take
> > a look:
> >
> > Here it is the benchmark code:
> >
> > (require 'cl-lib)
> > (require 'json)
> >
> > (defvar json-parsing-input (with-temp-buffer
> > (insert-file-contents-literally "benchmarks/sample.json")
> > (buffer-string)))
> >
> >
> > (defun elb-json-parsing-entry ()
> > (cl-loop repeat 1000
> > do (json-read-from-string json-parsing-input)))
> >
> > Thanks,
> > Ivan
>
> Hi Ivan,
>
> I've got time to re-look into this.
>
> The perf difference was due to GC time (as Eli pointed out all json
> computation is in the C world).
>
> I believe this delta was due to the greater number of objects allocated
> in the native comp variant with the consequent more time needed to
> traverse them during GC.
>
> Since these measures I've made several changes to reduce the number of
> unnecessary live objects. Repeating the test on the current native-comp
> against master I see on my machine a performance delta of 2% in favor of
> master. Considering I expect this effect to fade out on a real session
> with more allocated objects I consider this being acceptable.
>
> Andrea
>
> --
> akrl@sdf.org
>
[-- Attachment #2: Type: text/html, Size: 3457 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 13:05 ` Andrea Corallo
2019-12-26 17:30 ` Eli Zaretskii
@ 2019-12-26 20:26 ` Stefan Monnier
2019-12-26 20:58 ` dick.r.chiang
2019-12-26 21:57 ` Andrea Corallo
1 sibling, 2 replies; 210+ messages in thread
From: Stefan Monnier @ 2019-12-26 20:26 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> I'd be also very interested in maintainer's opinion.
I'm not the maintainer, but I find this work very interesting
and promising. I hope this can be put into at the very least a branch
in emacs.git and why not into Emacs itself later on.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-26 20:26 ` Stefan Monnier
@ 2019-12-26 20:58 ` dick.r.chiang
2019-12-26 21:57 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: dick.r.chiang @ 2019-12-26 20:58 UTC (permalink / raw)
To: Stefan Monnier; +Cc: emacs-devel, Andrea Corallo
Would like the how and who of branch instantiation in emacs.git.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 20:26 ` Stefan Monnier
2019-12-26 20:58 ` dick.r.chiang
@ 2019-12-26 21:57 ` Andrea Corallo
2019-12-27 8:22 ` Eli Zaretskii
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-26 21:57 UTC (permalink / raw)
To: Stefan Monnier; +Cc: emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>> I'd be also very interested in maintainer's opinion.
>
> I'm not the maintainer, but I find this work very interesting
> and promising. I hope this can be put into at the very least a branch
> in emacs.git and why not into Emacs itself later on.
Thanks,
I'm really unfamiliar with Emacs processes for this kind of case.
Is there any anything I could do to start the process for inclusion as a
branch into the Emacs repository?
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-26 21:57 ` Andrea Corallo
@ 2019-12-27 8:22 ` Eli Zaretskii
2019-12-27 10:49 ` Andrea Corallo
` (2 more replies)
0 siblings, 3 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-27 8:22 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Thu, 26 Dec 2019 21:57:07 +0000
> Cc: emacs-devel@gnu.org
>
> > I'm not the maintainer, but I find this work very interesting
> > and promising. I hope this can be put into at the very least a branch
> > in emacs.git and why not into Emacs itself later on.
>
> Thanks,
>
> I'm really unfamiliar with Emacs processes for this kind of case.
>
> Is there any anything I could do to start the process for inclusion as a
> branch into the Emacs repository?
Make sure your modifications are consistent with the Emacs build
procedure. The feature you are working on should be optional, invoked
by a configure-time option, say, --with-native-codegen, and any
system-dependent issues should generally be resolved by configure-time
tests. See configure.ac for how this is done for other features.
Once this is done (perhaps it already is, I don't know), create a
username on Savannah and request there to become a member of the Emacs
project. That will give you write access to the Emacs repository, and
you will be able to push a branch with your code. (Be sure to read
CONTRIBUTE for some of our coding and maintenance conventions first,
and if you are unsure how exactly to go about pushing a branch and how
to call it, ask here.)
TIA
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2019-12-27 8:22 ` Eli Zaretskii
@ 2019-12-27 10:49 ` Andrea Corallo
2019-12-28 16:10 ` dick.r.chiang
2019-12-31 10:24 ` Andrea Corallo
2 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-27 10:49 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Date: Thu, 26 Dec 2019 21:57:07 +0000
>> Cc: emacs-devel@gnu.org
>>
>> > I'm not the maintainer, but I find this work very interesting
>> > and promising. I hope this can be put into at the very least a branch
>> > in emacs.git and why not into Emacs itself later on.
>>
>> Thanks,
>>
>> I'm really unfamiliar with Emacs processes for this kind of case.
>>
>> Is there any anything I could do to start the process for inclusion as a
>> branch into the Emacs repository?
>
> Make sure your modifications are consistent with the Emacs build
> procedure. The feature you are working on should be optional, invoked
> by a configure-time option, say, --with-native-codegen, and any
> system-dependent issues should generally be resolved by configure-time
> tests. See configure.ac for how this is done for other features.
>
> Once this is done (perhaps it already is, I don't know), create a
> username on Savannah and request there to become a member of the Emacs
> project. That will give you write access to the Emacs repository, and
> you will be able to push a branch with your code. (Be sure to read
> CONTRIBUTE for some of our coding and maintenance conventions first,
> and if you are unsure how exactly to go about pushing a branch and how
> to call it, ask here.)
>
> TIA
Great thanks for precise info. I'll procede with that in the following
days then.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-27 8:22 ` Eli Zaretskii
2019-12-27 10:49 ` Andrea Corallo
@ 2019-12-28 16:10 ` dick.r.chiang
2019-12-28 17:31 ` Eli Zaretskii
2019-12-31 10:24 ` Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: dick.r.chiang @ 2019-12-28 16:10 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel, monnier, Andrea Corallo
The savannah emacs page points to
https://www.emacswiki.org/emacs/GitForEmacsDevs "for developer access."
There are several mentions of "if you are a committer", but no guidance on how
to become one.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-27 8:22 ` Eli Zaretskii
2019-12-27 10:49 ` Andrea Corallo
2019-12-28 16:10 ` dick.r.chiang
@ 2019-12-31 10:24 ` Andrea Corallo
2019-12-31 15:51 ` Eli Zaretskii
2020-01-01 14:59 ` Stefan Monnier
2 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2019-12-31 10:24 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> Once this is done (perhaps it already is, I don't know), create a
> username on Savannah and request there to become a member of the Emacs
> project. That will give you write access to the Emacs repository, and
> you will be able to push a branch with your code. (Be sure to read
> CONTRIBUTE for some of our coding and maintenance conventions first,
> and if you are unsure how exactly to go about pushing a branch and how
> to call it, ask here.)
>
> TIA
For the branch name would 'gccemacs' be fine?
Should I squash all my history or keep it as it is? The branch went
through quite some experimentations. I'm not sure about its history
value.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 10:24 ` Andrea Corallo
@ 2019-12-31 15:51 ` Eli Zaretskii
2019-12-31 16:21 ` Andrea Corallo
2020-01-01 14:59 ` Stefan Monnier
1 sibling, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-31 15:51 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Tue, 31 Dec 2019 10:24:37 +0000
>
> For the branch name would 'gccemacs' be fine?
The "emacs" part is redundant, so I'd suggest "libgccjit" instead.
> Should I squash all my history or keep it as it is? The branch went
> through quite some experimentations. I'm not sure about its history
> value.
It's completely up to you, since it's your history.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 15:51 ` Eli Zaretskii
@ 2019-12-31 16:21 ` Andrea Corallo
2019-12-31 16:47 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2019-12-31 16:21 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
>> Date: Tue, 31 Dec 2019 10:24:37 +0000
>>
>> For the branch name would 'gccemacs' be fine?
>
> The "emacs" part is redundant, so I'd suggest "libgccjit" instead.
The libgccjit interface and dependency is just a part of the realized
infrastructure (also in discussion). I'd like to call it something like
"native-comp" or "native-lisp". I believe is more descriptive of what
I've done and of its scope. Thinking about is not even a jit
compiler...
>
>> Should I squash all my history or keep it as it is? The branch went
>> through quite some experimentations. I'm not sure about its history
>> value.
>
> It's completely up to you, since it's your history.
If is possible to postpone the history rework to before the eventual
merge I guess I'll prefer to do so. It's a bit messy but if it does not
bother others it does not bother me.
Two branch related questions:
- Is force push allowed in these feature branches?
- Should it go in / /feature or /scratch ?
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 16:21 ` Andrea Corallo
@ 2019-12-31 16:47 ` Eli Zaretskii
2019-12-31 22:15 ` Óscar Fuentes
2020-01-01 12:41 ` Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: Eli Zaretskii @ 2019-12-31 16:47 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> Date: Tue, 31 Dec 2019 16:21:58 +0000
>
> Eli Zaretskii <eliz@gnu.org> writes:
>
> >> From: Andrea Corallo <akrl@sdf.org>
> >> Cc: monnier@iro.umontreal.ca, emacs-devel@gnu.org
> >> Date: Tue, 31 Dec 2019 10:24:37 +0000
> >>
> >> For the branch name would 'gccemacs' be fine?
> >
> > The "emacs" part is redundant, so I'd suggest "libgccjit" instead.
>
> The libgccjit interface and dependency is just a part of the realized
> infrastructure (also in discussion). I'd like to call it something like
> "native-comp" or "native-lisp". I believe is more descriptive of what
> I've done and of its scope. Thinking about is not even a jit
> compiler...
We are splitting hair, but...
"native-SOMETHING" is too general. E.g., we already had a serious
attempt to produce native code, by using libjit. We may have
additional attempts; for example Paul Eggert once proposed to teach
Emacs to generate C code from Lisp, and then compile it.
The advantage of libgccjit is that it pinpoints the main component on
which this particular implementation is based, so future attempts to
do it differently will not clash with the name.
That said, it's just a name. If you feel strongly about it, I won't
object.
> >> Should I squash all my history or keep it as it is? The branch went
> >> through quite some experimentations. I'm not sure about its history
> >> value.
> >
> > It's completely up to you, since it's your history.
>
> If is possible to postpone the history rework to before the eventual
> merge I guess I'll prefer to do so. It's a bit messy but if it does not
> bother others it does not bother me.
That's fine.
> - Is force push allowed in these feature branches?
For such a large feature I'd prefer not to do that, assuming that some
people will want to track that branch and perhaps contribute changes
and patches. Why would force-push be necessary?
> - Should it go in / /feature or /scratch ?
Under feature/, please.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 16:47 ` Eli Zaretskii
@ 2019-12-31 22:15 ` Óscar Fuentes
2020-01-01 12:41 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: Óscar Fuentes @ 2019-12-31 22:15 UTC (permalink / raw)
To: emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> "native-SOMETHING" is too general. E.g., we already had a serious
> attempt to produce native code, by using libjit. We may have
> additional attempts; for example Paul Eggert once proposed to teach
> Emacs to generate C code from Lisp, and then compile it.
Just for record, Tom Tromey also authored El Compilador, which does the
Elisp -> C thing:
https://github.com/tromey/el-compilador
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 16:47 ` Eli Zaretskii
2019-12-31 22:15 ` Óscar Fuentes
@ 2020-01-01 12:41 ` Andrea Corallo
2020-01-01 15:14 ` Stefan Monnier
2020-01-01 15:19 ` On elisp running native Óscar Fuentes
1 sibling, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-01 12:41 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> That said, it's just a name. If you feel strongly about it, I won't
> object.
Thanks you for your understanding.
>> - Is force push allowed in these feature branches?
>
> For such a large feature I'd prefer not to do that, assuming that some
> people will want to track that branch and perhaps contribute changes
> and patches. Why would force-push be necessary?
Just a curiosity. It can be used to keep on rebasing the branch fast
forward tho (thing I did till now) but is not good for cooperative
development I agree.
>
> Under feature/, please.
>
Okay, so the branch landed now on emacs.git as feature/native-comp.
I've freshly rebased it on the latest master (I just had to revert
186152ba4 but this is master related).
To test it configure it with --with-nativecomp (this is disabled by
default). I did some work and the normal build should be compilable and
functional again.
The build system produce elns as side effect of the elc compilation
during the boot for now. This is because there's no list of
lexical/dynamic scoped files. Is not the nicest but it works for now.
It bootstrap cleanly on two my machines. Obviously has to be considered
experimental and I've left comp-speed 0 as default setting.
Bootstrapping at speed 2 should work but has to be considered *meta*
experimental.
The question I wanted to ask is: What's next?
Other than rounding edges should be focus also on more performance
and/or dynamic scope support?
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 12:41 ` Andrea Corallo
@ 2020-01-01 15:14 ` Stefan Monnier
2020-01-01 18:42 ` Andrea Corallo
2020-01-02 11:47 ` Andrea Corallo
2020-01-01 15:19 ` On elisp running native Óscar Fuentes
1 sibling, 2 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-01-01 15:14 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
>> For such a large feature I'd prefer not to do that, assuming that some
>> people will want to track that branch and perhaps contribute changes
>> and patches. Why would force-push be necessary?
> Just a curiosity.
For completeness: force-push is technically disabled on the whole
repository. The only thing you can do technically instead is to delete
a branch and push a new branch at the same place (which ends up doing
the same).
We only tolerate such rebases on scratch/* branches.
> It can be used to keep on rebasing the branch fast forward tho (thing
> I did till now) but is not good for cooperative development I agree.
Merging works just as well as rebasing to stay in-sync with `master`.
And rebasing would be a pain in my rear when I start tracking your branch.
> Okay, so the branch landed now on emacs.git as feature/native-comp.
Great, thanks.
> Other than rounding edges should be focus also on more performance
> and/or dynamic scope support?
I think the focus should be on:
- making it work everywhere
- making it easy/convenient to use (e.g. even if you share your $HOME between
different architectures or even different OSes as well as different
versions of Emacs)
- making the code clean [ I haven't looked closely enough to know how much of
this might be necessary ]
- improve the quality of the generated code (size or speed, typically)
As far as I'm concerned, support for a nil value of lexical-binding is
a non-goal (it should still work, of course, e.g. by using the
bytecode): the dynbind version of Elisp is a legacy language and it's
better to update the code to lexbind than to try and make that legacy
code run faster.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-01 15:14 ` Stefan Monnier
@ 2020-01-01 18:42 ` Andrea Corallo
2020-01-02 7:55 ` arthur miller
` (2 more replies)
2020-01-02 11:47 ` Andrea Corallo
1 sibling, 3 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-01 18:42 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> I think the focus should be on:
> - making it work everywhere
> - making it easy/convenient to use (e.g. even if you share your $HOME between
> different architectures or even different OSes as well as different
> versions of Emacs)
I think the natural action would be to move the eln in the build
directory (as these are in fact compiled). Unfortunately this would not
work with the elpa packages... The other option would be to add a
suffix to the the eln file but is not nice at all. Any idea?
Consider that now the situation is pretty bad because eln are likely not
to be compatible even from different builds on the same codebase with
different configure parameters.
> As far as I'm concerned, support for a nil value of lexical-binding is
> a non-goal (it should still work, of course, e.g. by using the
> bytecode): the dynbind version of Elisp is a legacy language and it's
> better to update the code to lexbind than to try and make that legacy
> code run faster.
Good I agree, and even if not that hard would be something more to care
about.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* RE: On elisp running native
2020-01-01 18:42 ` Andrea Corallo
@ 2020-01-02 7:55 ` arthur miller
2020-01-02 11:31 ` Andrea Corallo
2020-01-04 18:17 ` Stefan Monnier
2020-01-02 13:47 ` Eli Zaretskii
2020-01-04 18:14 ` Stefan Monnier
2 siblings, 2 replies; 210+ messages in thread
From: arthur miller @ 2020-01-02 7:55 UTC (permalink / raw)
To: Andrea Corallo, Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel@gnu.org
[-- Attachment #1: Type: text/plain, Size: 2298 bytes --]
A question:
Would it be possible to lump all compiled functions into one .so file per package?
I have tested your branch after update2, haven't had time to test after the last update. I didn't see any speedup on load time, either with -Q flag, or when I loaded my init files and packages. I have complied all elpa packages to eln, including my custom lisp and init file (whatever was compilable).
I guess since there is one .eln per .el file, there is lots of disk access. Same for byte code and pure lisp. Maybe the init time is dominated by file access, rather then the processing those files.
Could it maybe be possible to create one .so per package si that all functions from corresponding eln files is that .so? It could drastically reduce number of file access at load time. I don't know if it is possible.
Skickat från min Samsung Galaxy-smartphone.
-------- Originalmeddelande --------
Från: Andrea Corallo <akrl@sdf.org>
Datum: 2020-01-01 19:42 (GMT+01:00)
Till: Stefan Monnier <monnier@iro.umontreal.ca>
Kopia: Eli Zaretskii <eliz@gnu.org>, emacs-devel@gnu.org
Ämne: Re: On elisp running native
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> I think the focus should be on:
> - making it work everywhere
> - making it easy/convenient to use (e.g. even if you share your $HOME between
> different architectures or even different OSes as well as different
> versions of Emacs)
I think the natural action would be to move the eln in the build
directory (as these are in fact compiled). Unfortunately this would not
work with the elpa packages... The other option would be to add a
suffix to the the eln file but is not nice at all. Any idea?
Consider that now the situation is pretty bad because eln are likely not
to be compatible even from different builds on the same codebase with
different configure parameters.
> As far as I'm concerned, support for a nil value of lexical-binding is
> a non-goal (it should still work, of course, e.g. by using the
> bytecode): the dynbind version of Elisp is a legacy language and it's
> better to update the code to lexbind than to try and make that legacy
> code run faster.
Good I agree, and even if not that hard would be something more to care
about.
--
akrl@sdf.org
[-- Attachment #2: Type: text/html, Size: 3471 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-02 7:55 ` arthur miller
@ 2020-01-02 11:31 ` Andrea Corallo
2020-01-02 13:53 ` Eli Zaretskii
2020-01-04 18:17 ` Stefan Monnier
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-01-02 11:31 UTC (permalink / raw)
To: arthur miller; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel@gnu.org
arthur miller <arthur.miller@live.com> writes:
> A question:
> Would it be possible to lump all compiled functions into one .so file per
> package?
> I have tested your branch after update2, haven't had time to test after
> the last update.
Thanks for the feedback, is cool to know that somebody reproduced.
> I didn't see any speedup on load time, either with -Q
> flag, or when I loaded my init files and packages. I have complied all
> elpa packages to eln, including my custom lisp and init file (whatever was
> compilable).
> I guess since there is one .eln per .el file, there is lots of disk
> access. Same for byte code and pure lisp. Maybe the init time is dominated
> by file access, rather then the processing those files.
> Could it maybe be possible to create one .so per package si that all
> functions from corresponding eln files is that .so? It could drastically
> reduce number of file access at load time. I don't know if it is
> possible.
I have no evidence that using eln is faster or slower than elc in terms
of load time. What you suggest is easily possible from the compiler
point of view but how should it behave exactly? We probably would need
some notion of package then?
Anyway I'd like to point out that:
- Mosts are likely to be compiling at speed 0 for now, and this
translate into considerably bigger elns (not sure this has an
impact).
- We should profile the load process first to understand were we spend
time. But I'm not sure we are at this stage of refinement.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-02 11:31 ` Andrea Corallo
@ 2020-01-02 13:53 ` Eli Zaretskii
0 siblings, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2020-01-02 13:53 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, arthur.miller, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: Stefan Monnier <monnier@iro.umontreal.ca>, Eli Zaretskii <eliz@gnu.org>,
> "emacs-devel\@gnu.org" <emacs-devel@gnu.org>
> Date: Thu, 02 Jan 2020 11:31:31 +0000
>
> - We should profile the load process first to understand were we spend
> time. But I'm not sure we are at this stage of refinement.
The function pdumper-stats provides the measured startup time, maybe
you could use that.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-02 7:55 ` arthur miller
2020-01-02 11:31 ` Andrea Corallo
@ 2020-01-04 18:17 ` Stefan Monnier
2020-01-05 1:33 ` Jean-Christophe Helary
1 sibling, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-01-04 18:17 UTC (permalink / raw)
To: arthur miller; +Cc: Eli Zaretskii, emacs-devel@gnu.org, Andrea Corallo
> I have tested your branch after update2, haven't had time to test after the
> last update. I didn't see any speedup on load time, either with -Q flag, or
> when I loaded my init files and packages.
I don't expect load time to be improved (just like loading a .elc file
isn't fundamentally any faster than loading the .el file (except for
the fact that .el files are currently always loaded in a somewhat
cumbersome way in order to perform character decoding according to the
file's coding-system)).
Lumping separate files one big one can likely be done in a way that's
largely independent from compilation issues (e.g. see package-quickstart).
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-04 18:17 ` Stefan Monnier
@ 2020-01-05 1:33 ` Jean-Christophe Helary
2020-01-05 4:51 ` Stefan Monnier
0 siblings, 1 reply; 210+ messages in thread
From: Jean-Christophe Helary @ 2020-01-05 1:33 UTC (permalink / raw)
To: emacs-devel@gnu.org
> On Jan 5, 2020, at 3:17, Stefan Monnier <monnier@iro.umontreal.ca> wrote:
>
> (except for
> the fact that .el files are currently always loaded in a somewhat
> cumbersome way in order to perform character decoding according to the
> file's coding-system)).
Can't it be enforced that they are all in one given encoding ?
Jean-Christophe Helary
-----------------------------------------------
http://mac4translators.blogspot.com @brandelune
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-05 1:33 ` Jean-Christophe Helary
@ 2020-01-05 4:51 ` Stefan Monnier
2020-01-05 5:18 ` Jean-Christophe Helary
0 siblings, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-01-05 4:51 UTC (permalink / raw)
To: Jean-Christophe Helary; +Cc: emacs-devel@gnu.org
> Can't it be enforced that they are all in one given encoding ?
You can
(setq load-source-file-function nil)
and then all .el files will be blindly treated as utf-8 (and loaded
faster). It'll work right with all .el files bundled with Emacs (and
all those in GNU ELPA as well, IIRC).
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-05 4:51 ` Stefan Monnier
@ 2020-01-05 5:18 ` Jean-Christophe Helary
2020-01-05 6:36 ` Stefan Monnier
0 siblings, 1 reply; 210+ messages in thread
From: Jean-Christophe Helary @ 2020-01-05 5:18 UTC (permalink / raw)
To: emacs-devel@gnu.org
> On Jan 5, 2020, at 13:51, Stefan Monnier <monnier@iro.umontreal.ca> wrote:
>
>> Can't it be enforced that they are all in one given encoding ?
>
> You can
>
> (setq load-source-file-function nil)
>
> and then all .el files will be blindly treated as utf-8 (and loaded
> faster). It'll work right with all .el files bundled with Emacs (and
> all those in GNU ELPA as well, IIRC).
But you wrote that ".el files are currently always loaded in a somewhat
cumbersome way in order to perform character decoding according to the
file's coding-system". Why isn't the emacs default to treat them all as utf-8 as you just suggested ?
Jean-Christophe Helary
-----------------------------------------------
http://mac4translators.blogspot.com @brandelune
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-05 5:18 ` Jean-Christophe Helary
@ 2020-01-05 6:36 ` Stefan Monnier
0 siblings, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-01-05 6:36 UTC (permalink / raw)
To: Jean-Christophe Helary; +Cc: emacs-devel@gnu.org
> But you wrote that ".el files are currently always loaded in a somewhat
> cumbersome way in order to perform character decoding according to the
> file's coding-system". Why isn't the emacs default to treat them all as
> utf-8 as you just suggested ?
Because Elisp files can use any coding system they like (some were
written before utf-8 was even supported by Emacs).
So Emacs has to look at the file first to see which coding system
it uses.
It can be improved, of course, but so far this hasn't been considered
important enough for anyone to spend time on it (tho I did go through
the trouble of converting Elisp files to utf-8 and changing Emacs such
that utf-8 is used by default (i.e. in the absence of evidence to the
contrary) in preference to defaulting to the locale, in the hope that
this decoding overhead can be eliminated in a decade or two).
Of course, instead of spending time on it, you can simply compile your
Elisp files: Emacs always uses the same coding-system for the .elc it
produces, so loading them can skip this overhead.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 18:42 ` Andrea Corallo
2020-01-02 7:55 ` arthur miller
@ 2020-01-02 13:47 ` Eli Zaretskii
2020-01-04 18:14 ` Stefan Monnier
2 siblings, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2020-01-02 13:47 UTC (permalink / raw)
To: Andrea Corallo; +Cc: monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Wed, 01 Jan 2020 18:42:02 +0000
> Cc: Eli Zaretskii <eliz@gnu.org>, emacs-devel@gnu.org
>
> Stefan Monnier <monnier@iro.umontreal.ca> writes:
>
> > I think the focus should be on:
> > - making it work everywhere
> > - making it easy/convenient to use (e.g. even if you share your $HOME between
> > different architectures or even different OSes as well as different
> > versions of Emacs)
>
> I think the natural action would be to move the eln in the build
> directory (as these are in fact compiled). Unfortunately this would not
> work with the elpa packages... The other option would be to add a
> suffix to the the eln file but is not nice at all. Any idea?
It might help to look at what Guile does when installing the *.go
files, and also how it handles the JIT compilation of the *.scm
files. AFAIU, the *.go files are also architecture-specific. Note
that Guile also has cache directories, both system-wide and per-user,
to support JIT compilation. This could give some ideas.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 18:42 ` Andrea Corallo
2020-01-02 7:55 ` arthur miller
2020-01-02 13:47 ` Eli Zaretskii
@ 2020-01-04 18:14 ` Stefan Monnier
2020-01-05 10:00 ` Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-01-04 18:14 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> I think the natural action would be to move the eln in the build
> directory (as these are in fact compiled). Unfortunately this would not
> work with the elpa packages...
Exactly, so it's not a good solution.
> The other option would be to add a suffix to the the eln file but is
> not nice at all. Any idea?
Actually, I think saving into a different directory is a better option.
E.g. just replacing /foo/bar.eln with /foo/<arch-vers>/bar.eln.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-04 18:14 ` Stefan Monnier
@ 2020-01-05 10:00 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-05 10:00 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>> I think the natural action would be to move the eln in the build
>> directory (as these are in fact compiled). Unfortunately this would not
>> work with the elpa packages...
>
> Exactly, so it's not a good solution.
>
>> The other option would be to add a suffix to the the eln file but is
>> not nice at all. Any idea?
>
> Actually, I think saving into a different directory is a better option.
> E.g. just replacing /foo/bar.eln with /foo/<arch-vers>/bar.eln.
Right, I think we need something like /foo/<arch-vers-hash>/bar.eln were
hash is computed using the list of Lisp primitives (that in the branch
we already accumulate into `comp-subr-list'). This depends on the
configure parameters and is effectively part of the ABI that Lisp
compiled code uses to plug into Emacs.
Actually hash could also take in account arch and vers but I think is
better to produce something as much human understandable as we can.
Also not sure how long is acceptable hash to be.
I'm thinking loud, hope it makes sense.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 15:14 ` Stefan Monnier
2020-01-01 18:42 ` Andrea Corallo
@ 2020-01-02 11:47 ` Andrea Corallo
2020-01-02 16:52 ` Ergus
2020-01-11 14:10 ` Andrea Corallo
1 sibling, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-02 11:47 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
> - making it work everywhere
I can report that the branch bootstraps out of the box on arm64!
I just have an assertion that triggers using some packages. But I get
the same on x86_64.
Blindly commenting it out with the following works but I've to find some
time to understand what's going on. I'm not into cl-generic.el therfore
suggestions are welcome :)
--
akrl@sdf.org
diff --git a/lisp/emacs-lisp/cl-generic.el b/lisp/emacs-lisp/cl-generic.el
index 4e8423eb5b..7aaec544bf 100644
--- a/lisp/emacs-lisp/cl-generic.el
+++ b/lisp/emacs-lisp/cl-generic.el
@@ -851,7 +851,7 @@ cl--generic-isnot-nnm-p
;; `next-method-p' just sucks, we should let it die. But EIEIO did support
;; it, and some packages use it, so we need to support it.
(catch 'found
- (cl-assert (function-equal cnm cl--generic-cnm-sample))
+ ;; (cl-assert (function-equal cnm cl--generic-cnm-sample))
(if (byte-code-function-p cnm)
(let ((cnm-constants (aref cnm 2))
(sample-constants (aref cl--generic-cnm-sample 2)))
^ permalink raw reply related [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-02 11:47 ` Andrea Corallo
@ 2020-01-02 16:52 ` Ergus
2020-01-02 17:55 ` Andrea Corallo
2020-01-11 14:10 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Ergus @ 2020-01-02 16:52 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Stefan Monnier, Eli Zaretskii, emacs-devel
Maybe a bit orthogonal to where the thread is going. But I have seen
that libgccjit is in alpha stage and has been like that for years
(according to the page). The source is included in gcc, but not compiled
by default in any distribution. I don't find distributions providing it
with the gcc bundle (I can ask for it's inclusion in arch, but I have no
idea about the others).
Andrea, are you aware about the progress of the development of libgccjit
and some estimation about when it is going to be stable? Because I can't
ask for the inclusion of alpha libraries in the distros.
Very thanks for this,
Ergus
On Thu, Jan 02, 2020 at 11:47:13AM +0000, Andrea Corallo wrote:
>Stefan Monnier <monnier@iro.umontreal.ca> writes:
>
>> - making it work everywhere
>
>I can report that the branch bootstraps out of the box on arm64!
>
>I just have an assertion that triggers using some packages. But I get
>the same on x86_64.
>
>Blindly commenting it out with the following works but I've to find some
>time to understand what's going on. I'm not into cl-generic.el therfore
>suggestions are welcome :)
>
>--
>akrl@sdf.org
>
>diff --git a/lisp/emacs-lisp/cl-generic.el b/lisp/emacs-lisp/cl-generic.el
>index 4e8423eb5b..7aaec544bf 100644
>--- a/lisp/emacs-lisp/cl-generic.el
>+++ b/lisp/emacs-lisp/cl-generic.el
>@@ -851,7 +851,7 @@ cl--generic-isnot-nnm-p
> ;; `next-method-p' just sucks, we should let it die. But EIEIO did support
> ;; it, and some packages use it, so we need to support it.
> (catch 'found
>- (cl-assert (function-equal cnm cl--generic-cnm-sample))
>+ ;; (cl-assert (function-equal cnm cl--generic-cnm-sample))
> (if (byte-code-function-p cnm)
> (let ((cnm-constants (aref cnm 2))
> (sample-constants (aref cl--generic-cnm-sample 2)))
>
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-02 16:52 ` Ergus
@ 2020-01-02 17:55 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-02 17:55 UTC (permalink / raw)
To: Ergus; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
Ergus <spacibba@aol.com> writes:
> Maybe a bit orthogonal to where the thread is going. But I have seen
> that libgccjit is in alpha stage and has been like that for years
> (according to the page). The source is included in gcc, but not compiled
> by default in any distribution. I don't find distributions providing it
> with the gcc bundle (I can ask for it's inclusion in arch, but I have no
> idea about the others).
AFAIK libgccjit is distributed by many (mosts?) distros including:
debian, fedora, ubuntu.
I guess this can easily improve further if Emacs choose to adopt it.
> Andrea, are you aware about the progress of the development of libgccjit
> and some estimation about when it is going to be stable? Because I can't
> ask for the inclusion of alpha libraries in the distros.
>
> Very thanks for this,
> Ergus
>
I don't know, in the last year I've up-streamed a bunch of patches to
extend it and keep it functional for gcc 9 and the upcoming 10 but I'm
not the maintainer. As a consequence regarding the alpha classification
I have no precise info. I invite you to ask on jit@gcc.gnu.org, I'll be
interested in reading the answer and I think is also important to
manifest the interest we have on that.
On the stability point: consider that libgccjit is a "relatively" thin
layer in front of the gcc pipeline. In my experience I can tell you
that the buggynes of this layer can typically impact the guy how is
writing the code generator. It means that if you are generating invalid
code and libgccjit does not catch it you'll get an ICE. It can be
annoying (unless one likes to work into gcc) but is effectively more of
a developer problem and if happen typically does not impact the final
user and the stability of the shipped code.
I think for Emacs adopting the use of libgccjit would help gcc giving
momentum for keep on having a good jitter and Emacs for being able to
use this direct interface into gcc. I see it as a win-win opportunity
for both and for GNU as a consequence.
That said if we decide to go for the other direction I've no problem in
investing some time to rewrite the final pass to target C instead.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-02 11:47 ` Andrea Corallo
2020-01-02 16:52 ` Ergus
@ 2020-01-11 14:10 ` Andrea Corallo
2020-02-04 11:28 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-01-11 14:10 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> Stefan Monnier <monnier@iro.umontreal.ca> writes:
>
>> - making it work everywhere
>
> I can report that the branch bootstraps out of the box on arm64!
>
> I just have an assertion that triggers using some packages. But I get
> the same on x86_64.
>
> Blindly commenting it out with the following works but I've to find some
> time to understand what's going on. I'm not into cl-generic.el therfore
> suggestions are welcome :)
Okay after some investigation I fixed the issue that was causing the
assertion in cl-generics to fire-up.
I wasn't moving the relocated lisp objects used by functions into pure
space during bootstrap. I pushed the fix and now should behave as the
standard implementation does.
Note this is an ABI change in the eln format so if you are testing the
branch and you have some eln compiled, for example in your elpa folder,
don't forget (as I did) to recompile.
Now I think it should be quite usable without any hacky trick.
Next I guess is to:
- implement the folder thing to have different Emacs versions to
coexists and use the same lisp file compiled.
- look into speed 2 to understand how stable is the bootstrapped
system and most likely do some bug fix.
For the rest I worked all the week on an arm64 Emacs to remember Friday
it was native compiled :)
I've a couple of interruptions that will slow me down for the next month
but I'll keep on progressing.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-01-11 14:10 ` Andrea Corallo
@ 2020-02-04 11:28 ` Andrea Corallo
2020-02-04 15:30 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-04 11:28 UTC (permalink / raw)
To: emacs-devel
Hi all,
I did some bug fixing and as a result speed 1 should be okay now. Still
I've left 0 as default for now. I'll keep on working towards verifying
and stabilizing speed 2 too.
By the way I've tried to rebase on latest trunk and did some boostraps
but I get from time to time backtraces always going through libpthread.
Does this ring any bell to anybody? The branch the native compiler is
currently based on date beginning of the year. Did something
fundamental changed around threading?
These fails are not reproducible so suggestions are welcome otherwise
I'll look into them with rr.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 11:28 ` Andrea Corallo
@ 2020-02-04 15:30 ` Eli Zaretskii
2020-02-04 18:39 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-02-04 15:30 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc:
> Date: Tue, 04 Feb 2020 11:28:19 +0000
>
> By the way I've tried to rebase on latest trunk and did some boostraps
> but I get from time to time backtraces always going through libpthread.
Can you show an example of such a backtrace, with source file,
function name, and line number information?
> Does this ring any bell to anybody? The branch the native compiler is
> currently based on date beginning of the year. Did something
> fundamental changed around threading?
Bootstrapping Emacs doesn't use threads, so I'm unsure what changes in
threading could be relevant.
But to answer your question, we changed slightly how thread is
started, as side effect of fixing the code which assigns a name to a
new thread.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 15:30 ` Eli Zaretskii
@ 2020-02-04 18:39 ` Andrea Corallo
2020-02-04 18:58 ` Eli Zaretskii
2020-02-04 19:00 ` Paul Eggert
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-02-04 18:39 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc:
>> Date: Tue, 04 Feb 2020 11:28:19 +0000
>>
>> By the way I've tried to rebase on latest trunk and did some boostraps
>> but I get from time to time backtraces always going through libpthread.
>
> Can you show an example of such a backtrace, with source file,
> function name, and line number information?
>
>> Does this ring any bell to anybody? The branch the native compiler is
>> currently based on date beginning of the year. Did something
>> fundamental changed around threading?
>
> Bootstrapping Emacs doesn't use threads, so I'm unsure what changes in
> threading could be relevant.
>
> But to answer your question, we changed slightly how thread is
> started, as side effect of fixing the code which assigns a name to a
> new thread.
Hi Eli,
the backtraces I've observed are like this:
Fatal error 11: Segmentation fault
Backtrace:
../src/bootstrap-emacs[0x5a7462]
../src/bootstrap-emacs[0x55fbe8]
../src/bootstrap-emacs[0x5a6d01]
../src/bootstrap-emacs[0x5a6cd6]
../src/bootstrap-emacs[0x5a6d38]
../src/bootstrap-emacs[0x5a6ed2]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x12890)[0x7f5d482cf890]
../src/bootstrap-emacs[0x512f19]
../src/bootstrap-emacs[0x6dbb91]
they are not run into the debugger so there's no line number (even if
bootstrap-emacs has debug symbols).
Anyway at this point I suspect is not thread related and is likely to be
my fault. Tonight I'll run some tests under rr to better understand
what's going on thanks.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 18:39 ` Andrea Corallo
@ 2020-02-04 18:58 ` Eli Zaretskii
2020-02-04 19:00 ` Paul Eggert
1 sibling, 0 replies; 210+ messages in thread
From: Eli Zaretskii @ 2020-02-04 18:58 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: emacs-devel@gnu.org
> Date: Tue, 04 Feb 2020 18:39:38 +0000
>
> the backtraces I've observed are like this:
>
> Fatal error 11: Segmentation fault
> Backtrace:
> ../src/bootstrap-emacs[0x5a7462]
> ../src/bootstrap-emacs[0x55fbe8]
> ../src/bootstrap-emacs[0x5a6d01]
> ../src/bootstrap-emacs[0x5a6cd6]
> ../src/bootstrap-emacs[0x5a6d38]
> ../src/bootstrap-emacs[0x5a6ed2]
> /lib/x86_64-linux-gnu/libpthread.so.0(+0x12890)[0x7f5d482cf890]
> ../src/bootstrap-emacs[0x512f19]
> ../src/bootstrap-emacs[0x6dbb91]
>
> they are not run into the debugger so there's no line number (even if
> bootstrap-emacs has debug symbols).
The Emacs manual explains how to convert these backtraces to the
information I'd like to see. See the node "Crashing" there.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 18:39 ` Andrea Corallo
2020-02-04 18:58 ` Eli Zaretskii
@ 2020-02-04 19:00 ` Paul Eggert
2020-02-04 20:30 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Paul Eggert @ 2020-02-04 19:00 UTC (permalink / raw)
To: Andrea Corallo, Eli Zaretskii; +Cc: emacs-devel
On 2/4/20 10:39 AM, Andrea Corallo wrote:
> they are not run into the debugger so there's no line number (even if
> bootstrap-emacs has debug symbols).
If you have debug symbols you can get more info via addr2line as
described here:
https://www.gnu.org/software/emacs/manual/html_node/emacs/Crashing.html
Running under a debugger is better, though.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 19:00 ` Paul Eggert
@ 2020-02-04 20:30 ` Andrea Corallo
2020-02-05 10:15 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-04 20:30 UTC (permalink / raw)
To: Paul Eggert; +Cc: Eli Zaretskii, emacs-devel
Paul Eggert <eggert@cs.ucla.edu> writes:
> On 2/4/20 10:39 AM, Andrea Corallo wrote:
>> they are not run into the debugger so there's no line number (even if
>> bootstrap-emacs has debug symbols).
>
> If you have debug symbols you can get more info via addr2line as
> described here:
>
> https://www.gnu.org/software/emacs/manual/html_node/emacs/Crashing.html
>
> Running under a debugger is better, though.
Thanks both for the suggestion. I knew about addr2line but never
thought about using it with sed to convert backtraces! (also this signal
I should read more the manual).
I don't have that executable anymore but this will be certainly useful.
Certainly know already but alternatively just compiling with -rdynamic
in the CFLAGS we can have glibc decoding and printing the backtrace all
by himself.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-04 20:30 ` Andrea Corallo
@ 2020-02-05 10:15 ` Andrea Corallo
2020-02-05 14:24 ` Stefan Monnier
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-05 10:15 UTC (permalink / raw)
To: Paul Eggert; +Cc: Eli Zaretskii, emacs-devel
Looking at tonight tests I've no evidence the merge is armful. At this
point I'm not sure what I was really observing the other day. I guess
was unrelated but anyway if there's a problem we'll notice it because
I've pushed the merge commit into feature/native-comp (I guess is
important to stay in sync).
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-05 10:15 ` Andrea Corallo
@ 2020-02-05 14:24 ` Stefan Monnier
2020-02-15 9:29 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-02-05 14:24 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Paul Eggert, emacs-devel
> I've pushed the merge commit into feature/native-comp
Thanks.
> (I guess is important to stay in sync).
Yes, if we want to include it into `master` at some point in the future,
it's important to keep it updated (even though it may take a fairly
long time: I do hope it will be significantly less time than it took
for lexbind, but still).
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-02-05 14:24 ` Stefan Monnier
@ 2020-02-15 9:29 ` Andrea Corallo
2020-02-16 18:18 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-15 9:29 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier
All right, small update:
- I've redesigned the variables layout in the generated code. Now every
call by reference gets its own unique array for its arguments. I
believe this is the right way to express to the compiler what is going
on.
This has no impact on the compilation time, but produced binaries are
slightly smaller.
I'm pushing it after having rebased on top of master.
- I think is worth mentioning that is now 2 weeks circa that me and a
colleague are using for everyday production Emacs all compiled at
speed 2. So far we have seen no crashes.
I consider having speed 2 stable a major achievement.
- Connected to the previous point with the today push I'm setting speed
2 as default. This blows up compilation time to ~7h CPU time on my
laptop (to be divided by the number of threads). We can always
fallback to speed 0 (~1h) but I think its important to test with
optimizations.
Also I took some measures, around 25% of the total compile time goes
into leim. ZIRANMA.el take like ~50min. Still have to understand
exactly what's going on with these but it's interesting for
speculation purpose given the specific kind of file :)
Also I'm questioning my self it's really worth compiling these files.
- I still have to investigate the fact that the pure space usage appears
to be a little higher when native compiling. In case some special
trick to look into this already exists please mention.
Andrea
PS I've also tracked down the observed crashes I previously mentioned.
I tested two versions of the same patch convinced they were equivalent
but wasn't the case, my fault :/
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-15 9:29 ` Andrea Corallo
@ 2020-02-16 18:18 ` Andrea Corallo
2020-02-22 10:15 ` On elisp running native - deferred compilation Andrea Corallo
2020-03-04 19:34 ` On elisp running native Andrea Corallo
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-02-16 18:18 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier
Andrea Corallo <akrl@sdf.org> writes:
> All right, small update:
> ...
To complete with today's push:
I added a simple pass to perform self Tail Call Optimization.
It was relatively easy to add and I think is nice to have. This is now
gated at speed 3 being very fresh.
I've summarized what I've pushed this weekend in a new update on the
usual https://akrl.sdf.org/gccemacs.html. I think is good to keep a
chronological track of the progresses.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* On elisp running native - deferred compilation
2020-02-16 18:18 ` Andrea Corallo
@ 2020-02-22 10:15 ` Andrea Corallo
2020-02-22 14:19 ` Stefan Monnier
2020-03-04 19:34 ` On elisp running native Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-02-22 10:15 UTC (permalink / raw)
To: emacs-devel
Hi all,
It's a couple of days that an idea is spinning in my head. I think we
could imagine the following mechanism:
1- When an .elc is loaded when no .eln is present we load the .elc but
we enqeue an async compilation using 'native-compile-async'. The async
compilation queue is eaten by a configurable number of parallel
processes.
2- When each compilation unit is finished compiling we substitute each
byte-compiled function definition with the new one.
I think we already have most of the infrastructure for that.
The missing (and a little tricky) part is that we would have to avoid
redefining functions that after the first load got already redefined for
some reason.
In a sense we would have something like a jitter (with a per compilation
unit granularity) but in contrast with the conventional jitter approach
we would have .eln already compiled at the next run avoiding having to
boil the same water again.
I think such mechanism would work very well when installing packages
without having to freeze for compiling. But we could also introduce a
new build target where we native compile only preloaded files. This
would bring down compile time of one magnitude order.
It's just a (possibly completely broken) idea so I'd like to hear
opinions.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - deferred compilation
2020-02-22 10:15 ` On elisp running native - deferred compilation Andrea Corallo
@ 2020-02-22 14:19 ` Stefan Monnier
0 siblings, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-02-22 14:19 UTC (permalink / raw)
To: Andrea Corallo; +Cc: emacs-devel
> 1- When an .elc is loaded when no .eln is present we load the .elc but
> we enqeue an async compilation using 'native-compile-async'. The async
> compilation queue is eaten by a configurable number of parallel
> processes.
> 2- When each compilation unit is finished compiling we substitute each
> byte-compiled function definition with the new one.
Sounds good.
I'll just mention that IMO, we should move towards a model where
compilation (byte or native or anything else) is performed more
transparently and systematically:
- Elisp source code is always compiled before being executed: currently
the semantics of loading a file allows the behavior to differ between
loading the .el and loading the .elc, whereas I think it should always
behave like loading a .elc.
The eager-macroexpansion introduced in Emacs-24 was a step in
that direction.
- Recompilation happens automatically (e.g. either because the source
Elisp file was modified or because the compiled code doesn't match the
currently running Emacs version).
This also suggests that compiled files should be "hidden" and include
some kind of version info.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-02-16 18:18 ` Andrea Corallo
2020-02-22 10:15 ` On elisp running native - deferred compilation Andrea Corallo
@ 2020-03-04 19:34 ` Andrea Corallo
2020-03-04 20:21 ` Michael Albinus
` (2 more replies)
1 sibling, 3 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-04 19:34 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier
Hi all,
this is the usual update for the activity of the last 2/3 weeks on the
native-comp branch.
- I did some reorganization of the passes and reworked the allocation
strategy for the relocable objects to avoid duplication and save heap
space.
- We now (finally) support multiple different builds on the same lisp
code base.
~eln~ files are compiled in specific sub-folders taking in account
host architecture and current Emacs configuration to disambiguate
possible incompatibilities.
As example a file ~.../foo.el~ will compile into something like
~.../x86_64-pc-linux-gnu-12383a81d4f33a87/foo.eln~.
I've updated the load and 'go to definition' mechanisms to support
this, so should be totally transparent to the user (if it's not is a
bug).
The same idea applies to all compilations, meaning main build as
compiled packages.
- Optimize qualities (read compilation flags) are now stored in every
~eln~ file. It's possible to inspect them for example as:
#+BEGIN_SRC lisp
(subr-native-comp-unit (symbol-function 'org-mode)) =>
#<native compilation unit: .../emacs/lisp/org/x86_64-pc-linux-gnu-12383a81d4f33a87/org.eln ((comp-speed . 2) (comp-debug . 0))>
#+END_SRC
We could store there other information we are interest in related to
the compilation unit if we need.
- Finally I've set up a docker image for people willing to test it
without having to configure and compile Emacs and libgccjit. Now that
I've learned how it works I plan to use it to setup a small CI to keep
an eye that the latest GCC does not break with our build.
I've took note of all of this as Update6 in the usual page.
I'm generally very satisfied (surprised) about the stability of the toy.
I'm looking forward going into compile time mitigation. I guess I'll
start this weekend with deferred compilation and GCC profiling.
Please let me know if there's some point I should address.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-04 19:34 ` On elisp running native Andrea Corallo
@ 2020-03-04 20:21 ` Michael Albinus
2020-03-04 22:16 ` Andrea Corallo
2020-03-05 10:54 ` Adam Porter
2020-03-23 21:36 ` On elisp running native - Update7 Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: Michael Albinus @ 2020-03-04 20:21 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> Hi all,
Hi Andrea,
> - Finally I've set up a docker image for people willing to test it
> without having to configure and compile Emacs and libgccjit. Now that
> I've learned how it works I plan to use it to setup a small CI to keep
> an eye that the latest GCC does not break with our build.
I don't know whether you are familiar with gitlab's CI. In case you are,
and you want to use it: Emacs runs an own gitlab instance on
<https://emba.gnu.org>. You'll see there that your last commit has
failed the CI test, see <https://emba.gnu.org/emacs/emacs/-/jobs/5147/raw>.
Besides fixing this, you could also *add* own CI tests, which will be
run once your branch has seen any git push. The tests are described in
Emacs' root directory, file .gitlab-ci.yml. Read also admin/notes/emba.
Just in case you're interested in.
> Andrea
Best regards, Michael.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-04 20:21 ` Michael Albinus
@ 2020-03-04 22:16 ` Andrea Corallo
2020-03-05 19:33 ` Michael Albinus
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-03-04 22:16 UTC (permalink / raw)
To: Michael Albinus; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Michael Albinus <michael.albinus@gmx.de> writes:
> I don't know whether you are familiar with gitlab's CI. In case you are,
> and you want to use it: Emacs runs an own gitlab instance on
> <https://emba.gnu.org>. You'll see there that your last commit has
> failed the CI test, see <https://emba.gnu.org/emacs/emacs/-/jobs/5147/raw>.
Hi Michael,
thanks for the info, these prove to be already very useful! :)
I didn't know the CI was running all branches. As I'm rarely building
the non native configuration on that branch (unless when doing
comparisons) I'll certanly keep a look on it since today.
Should be fixed by now lets see if it pass then.
> Besides fixing this, you could also *add* own CI tests, which will be
> run once your branch has seen any git push. The tests are described in
> Emacs' root directory, file .gitlab-ci.yml. Read also admin/notes/emba.
>
> Just in case you're interested in.
I'd be certanly very interested. The issue could be that, as of today,
bootstrapping and native compiling the whole Emacs is about 7h total cpu
time (to be divided by the machine parallelism). Not sure how many cpu
threads emba.gnu.org has and if we want to load it with this.
Regards
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-04 22:16 ` Andrea Corallo
@ 2020-03-05 19:33 ` Michael Albinus
2020-03-06 21:44 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Michael Albinus @ 2020-03-05 19:33 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> Hi Michael,
Hi Andrea,
>> Besides fixing this, you could also *add* own CI tests, which will be
>> run once your branch has seen any git push. The tests are described in
>> Emacs' root directory, file .gitlab-ci.yml. Read also admin/notes/emba.
>>
>> Just in case you're interested in.
>
> I'd be certanly very interested. The issue could be that, as of today,
> bootstrapping and native compiling the whole Emacs is about 7h total cpu
> time (to be divided by the machine parallelism). Not sure how many cpu
> threads emba.gnu.org has and if we want to load it with this.
We could use Pipeline Schedules for that, see
<https://docs.gitlab.com/ee/user/project/pipelines/schedules.html>. Running
bootstrapping once a week or once a month, as needed. It could still be
triggered manually, in case you want to test the CI integration itself.
emba.gnu.org is a VM. It reports 2 cores / 8GB RAM. I have no idea how
your full bootstrapping fits, we'll see.
> Regards
>
> Andrea
Best regards, Michael.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 19:33 ` Michael Albinus
@ 2020-03-06 21:44 ` Andrea Corallo
2020-03-07 9:09 ` Michael Albinus
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-03-06 21:44 UTC (permalink / raw)
To: Michael Albinus; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Michael Albinus <michael.albinus@gmx.de> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> Hi Michael,
>
> Hi Andrea,
>
>>> Besides fixing this, you could also *add* own CI tests, which will be
>>> run once your branch has seen any git push. The tests are described in
>>> Emacs' root directory, file .gitlab-ci.yml. Read also admin/notes/emba.
>>>
>>> Just in case you're interested in.
>>
>> I'd be certanly very interested. The issue could be that, as of today,
>> bootstrapping and native compiling the whole Emacs is about 7h total cpu
>> time (to be divided by the machine parallelism). Not sure how many cpu
>> threads emba.gnu.org has and if we want to load it with this.
>
> We could use Pipeline Schedules for that, see
> <https://docs.gitlab.com/ee/user/project/pipelines/schedules.html>. Running
> bootstrapping once a week or once a month, as needed. It could still be
> triggered manually, in case you want to test the CI integration itself.
>
> emba.gnu.org is a VM. It reports 2 cores / 8GB RAM. I have no idea how
> your full bootstrapping fits, we'll see.
>
>> Regards
>>
>> Andrea
>
> Best regards, Michael.
>
Hi Michael,
I took a look and seen that the debian container we are using for
testing has libgccjit6 available as package.
Being this debian stretch I don't understand why it's not libgccjit 7 or
8 looking at:
https://packages.debian.org/search?suite=buster&arch=amd64&searchon=names&keywords=libgccjit
but anyway... I did some tests in the docker and apparently it works!
(we are lucky often libgccjit distributed can be broken).
I added a test to the yml that performs a native bootstrap:
https://gitlab.com/koral/gccemacs/-/commit/8d6fb91c7448f438744dec8f3cc9a51d0e1d8254
Now what is going to happen if I push this? Is this test going to run
every time I push or we have a way to select the tests to be run per
test?
Also if I try to look into the pipelines in emba following
https://emba.gnu.org/emacs/emacs/pipeline_schedules I get 404. Is this
expected?
Sorry as it's clear I'm not really skilled into this.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-06 21:44 ` Andrea Corallo
@ 2020-03-07 9:09 ` Michael Albinus
2020-03-07 9:33 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Michael Albinus @ 2020-03-07 9:09 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> Hi Michael,
Hi Andrea,
> I took a look and seen that the debian container we are using for
> testing has libgccjit6 available as package.
>
> Being this debian stretch I don't understand why it's not libgccjit 7 or
> 8 looking at:
>
> https://packages.debian.org/search?suite=buster&arch=amd64&searchon=names&keywords=libgccjit
I guess if needed you could install the preferred libgccjit yourself in
the spec.
> but anyway... I did some tests in the docker and apparently it works!
> (we are lucky often libgccjit distributed can be broken).
>
> I added a test to the yml that performs a native bootstrap:
>
> https://gitlab.com/koral/gccemacs/-/commit/8d6fb91c7448f438744dec8f3cc9a51d0e1d8254
>
> Now what is going to happen if I push this? Is this test going to run
> every time I push or we have a way to select the tests to be run per
> test?
You can declare it yourself. Change it to
--8<---------------cut here---------------start------------->8---
test-native-bootstrap:
# Test native bootstrap
stage: test
only:
- schedules
script:
- DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 libgccjit-6-dev
- ./autogen.sh autoconf
- ./configure --without-makeinfo --with-nativecomp
- make bootstrap
--8<---------------cut here---------------end--------------->8---
I have removed the inotify-tools, you don't need them I guess. And I
have added the "only:\n - schedules" lines, which means it runs only
when scheduled.
For your branch I have created a schedule which runs every Sun on
4am. Let's see how it goes, we can change it as needed.
> Also if I try to look into the pipelines in emba following
> https://emba.gnu.org/emacs/emacs/pipeline_schedules I get 404. Is this
> expected?
Yes, you need an account on emba to see the schedules. I've created the
user "corallo" on emba.gnu.org for you, you should have received an
email asking you to set your password.
Maybe you just push your changes to the Emacs master repo?
> Thanks
>
> Andrea
Best regards, Michael.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-07 9:09 ` Michael Albinus
@ 2020-03-07 9:33 ` Andrea Corallo
2020-03-07 9:41 ` Michael Albinus
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-03-07 9:33 UTC (permalink / raw)
To: Michael Albinus; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Michael Albinus <michael.albinus@gmx.de> writes:
> You can declare it yourself. Change it to
>
> test-native-bootstrap:
> # Test native bootstrap
> stage: test
> only:
> - schedules
> script:
> - DEBIAN_FRONTEND=noninteractive apt install --no-install-recommends -y -qq -o=Dpkg::Use-Pty=0 libgccjit-6-dev
> - ./autogen.sh autoconf
> - ./configure --without-makeinfo --with-nativecomp
> - make bootstrap
>
> I have removed the inotify-tools, you don't need them I guess. And I
> have added the "only:\n - schedules" lines, which means it runs only
> when scheduled.
>
> For your branch I have created a schedule which runs every Sun on
> 4am. Let's see how it goes, we can change it as needed.
Cool. I was thinking that if it takes too long we could bootstrap speed
0, is way faster and is good to test it too.
>
>> Also if I try to look into the pipelines in emba following
>> https://emba.gnu.org/emacs/emacs/pipeline_schedules I get 404. Is this
>> expected?
>
> Yes, you need an account on emba to see the schedules. I've created the
> user "corallo" on emba.gnu.org for you, you should have received an
> email asking you to set your password.
Great
> Maybe you just push your changes to the Emacs master repo?
Done
>> Thanks
>>
>> Andrea
>
> Best regards, Michael.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-04 19:34 ` On elisp running native Andrea Corallo
2020-03-04 20:21 ` Michael Albinus
@ 2020-03-05 10:54 ` Adam Porter
2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
` (2 more replies)
2020-03-23 21:36 ` On elisp running native - Update7 Andrea Corallo
2 siblings, 3 replies; 210+ messages in thread
From: Adam Porter @ 2020-03-05 10:54 UTC (permalink / raw)
To: emacs-devel
Hi Andrea,
Andrea Corallo <akrl@sdf.org> writes:
> - Finally I've set up a docker image for people willing to test it
> without having to configure and compile Emacs and libgccjit. Now that
> I've learned how it works I plan to use it to setup a small CI to keep
> an eye that the latest GCC does not break with our build.
Thank you for setting up the Docker image. I had tried to build the
native-comp branch on my system but finally was unable to overcome a
weird error related to libgccjit (probably because I need to upgrade my
system). I even tried using Guix to build it, but libgccjit isn't
packaged in it (and I'm completely new to Guix packaging, so I wasn't
able to figure out how to adjust the GCC build for that in a reasonable
amount of time).
I managed to get the Docker image working, and I did a little testing
with one of my packages, org-ql, which is for searching Org files. I
installed it and its dependencies, then used native-comp-async to
compile all of ~/.emacs.d/elpa. Then I restarted Emacs to ensure that
the native libraries were loaded, and finally I ran the following code
and observed a more than 2x improvement!
#+BEGIN_SRC elisp
;; Reset the cache to force search to run again.
(setf org-ql-cache (make-hash-table :test #'equal))
;; Set my personal groups (omitted) and enable org-super-agenda-mode
;; to display in groups.
(setf org-super-agenda-groups my-groups)
(org-super-agenda-mode)
;; Load all my Org files and search them.
(let* ((enable-local-variables nil)
(files (directory-files org-directory 'full (rx ".org" eos))))
(dolist (file files)
(find-file-noselect file 'nowarn))
(benchmark-run-compiled
(org-ql-search files
;; The (tags) selector tends to be slow compared to other
;; predicates because of searching the hierarchy repeatedly
;; and tag inheritance, so it makes for an interesting test.
'(and (todo) (tags "Emacs"))
:super-groups org-super-agenda-groups)))
#+END_SRC
#+RESULTS:
| 5.819892666 | 1 | 0.6101534569999956 | In Emacs 26.3
| 2.317804428 | 3 | 0.6223487619999997 | In Emacs 28 with native-comp
Then I tested this more complex query and observed an even more
impressive speedup of about 3.4x!
'(and (not (done))
(or (habit)
(deadline auto)
(scheduled :to today)
(ts-active :on today)))
#+RESULTS:
| 8.377843199 | 1 | 0.6277314330000081 | In Emacs 26.3
| 2.436048375 | 2 | 0.4278436059999997 | In Emacs 28 with native-comp
This is very exciting! The difference between 8.4 seconds and 2.4
seconds is a huge usability improvement.
And everything seems to work fine! No bugs, errors, or crashes of any
kind.
I did encounter a minor issue related to macro-expansion and loading
that may be my fault. When I tried to (require 'org-ql) or call a
function that is autoloaded from it, I got an error saying that the .eln
file did not provide the org-ql feature. I surmised that meant that the
file hadn't been compiled properly, so I looked at the async compilation
log and saw an error saying that the variable org-ql-predicates was not
defined. At this URL you can see a macro definition,
org-ql--def-plain-query-fn, and the call to it that's wrapped in
cl-eval-when to ensure it works properly with the byte-compiler:
https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L1671
The macro uses the value of org-ql-predicates, which is defined here:
https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L126
But when the org-ql--defpred macro:
https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L140
...is called, it adds to that variable. So all of that works with the
byte-compiler, but apparently not with the native compilation.
To work around it, I wrapped (defvar org-ql-predicates...) in
eval-and-compile, and then wrapped all of the calls to org-ql--defpred
in eval-and-compile as well. Then I deleted the org-ql.eln file and
recompiled it, restarted Emacs, and then everything worked fine.
I'm guessing that this issue is "native" to the native-compilation and
might need to be documented in some way, but I wouldn't expect it to
affect many packages, only ones that do tricky things with variables and
macro-expansion like this.
So, a few ideas for fixes or improvements:
1. I guess the .eln file should not have been produced or left behind
in the directory, since there was a compilation error.
2. The compilation error should probably be shown as a warning or
something, because it's easily lost in the async compilation log
buffer.
3. It would probably be good if the loader fell back to .elc and .el
files if the .eln file doesn't seem to work, and showed a warning
when doing so. The library would remain usable in its byte-compiled
form, and in the case of a package like this, the user could then
report the native-compilation error to the developer.
Other than that minor issue, everything works and feels very fast! This
is great!
> ~eln~ files are compiled in specific sub-folders taking in account
> host architecture and current Emacs configuration to disambiguate
> possible incompatibilities.
> As example a file ~.../foo.el~ will compile into something like
> ~.../x86_64-pc-linux-gnu-12383a81d4f33a87/foo.eln~.
FYI, I did not see the .eln files being put in subdirectories like that
in the Docker image. Maybe it didn't receive that update yet.
This is all very exciting. I'm imagining an update to package.el that
would byte-compile packages immediately, as usual, then start
native-compile-async'ing them in the background, loading each file's
native version as it completes, with the packages being immediately
usable and seeming to get faster as each file is compiled and loaded.
> I'm generally very satisfied (surprised) about the stability of the
> toy. I'm looking forward going into compile time mitigation. I guess
> I'll start this weekend with deferred compilation and GCC profiling.
I think it isn't a toy anymore! :) Thanks very much for your work on
this. Exciting times for Emacs, as this feature, in one form or
another, has been eagerly awaited for years.
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-05 10:54 ` Adam Porter
@ 2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
2020-03-05 13:27 ` Andrea Corallo
2020-03-05 13:19 ` Andrea Corallo
2020-03-11 22:04 ` Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: Gong-Yi Liao 廖宮毅 @ 2020-03-05 12:43 UTC (permalink / raw)
To: Adam Porter; +Cc: emacs-devel
That's pretty good on (I assume) x86_64, but native-comp branch seems
not working on ARM64 Debian Buster at this moment:
Not sure if it's a GCCJIT issue or the native-comp branch issue:
Any comments on this compilation log are appreciated.
Thanks,
Gong-Yi.
------
make -C ../lisp compile-first EMACS="../src/bootstrap-emacs"
make[2]: Entering directory '/home/pi/Downloads/emacs/master/lisp'
ELC+ELN emacs-lisp/comp.elc
ELC+ELN emacs-lisp/bytecomp.elc
ELC+ELN emacs-lisp/autoload.elc
Backtrace:
../src/bootstrap-emacs(+0x1289f0)[0x555c89f9f0]
../src/bootstrap-emacs(+0x46b5c)[0x555c7bdb5c]
../src/bootstrap-emacs(+0x46d50)[0x555c7bdd50]
../src/bootstrap-emacs(+0x12720c)[0x555c89e20c]
../src/bootstrap-emacs(+0x127298)[0x555c89e298]
linux-vdso.so.1(__kernel_rt_sigreturn+0x0)[0x7fa303b698]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc014dc)[0x7f9c8b94dc]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc015b0)[0x7f9c8b95b0]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17c890)[0x7f9be34890]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17ec94)[0x7f9be36c94]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17f5c8)[0x7f9be375c8]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17e760)[0x7f9be36760]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x1749e8)[0x7f9be2c9e8]
/lib/aarch64-linux-gnu/libgccjit.so.0(gcc_jit_context_compile_to_file+0x94)[0x7f9be214ec]
../src/bootstrap-emacs(+0x1b9bbc)[0x555c930bbc]
../src/bootstrap-emacs(+0x18119c)[0x555c8f819c]
../src/bootstrap-emacs(+0x1818ac)[0x555c8f88ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x1825ac)[0x555c8f95ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x18160c)[0x555c8f860c]
../src/bootstrap-emacs(+0x180cc8)[0x555c8f7cc8]
../src/bootstrap-emacs(+0x180f7c)[0x555c8f7f7c]
../src/bootstrap-emacs(+0x181a40)[0x555c8f8a40]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x182864)[0x555c8f9864]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x1825ac)[0x555c8f95ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x18160c)[0x555c8f860c]
../src/bootstrap-emacs(+0x17f0fc)[0x555c8f60fc]
../src/bootstrap-emacs(+0x1812a0)[0x555c8f82a0]
../src/bootstrap-emacs(+0x181a40)[0x555c8f8a40]
../src/bootstrap-emacs(+0x1810c4)[0x555c8f80c4]
../src/bootstrap-emacs(+0x18160c)[0x555c8f860c]
../src/bootstrap-emacs(+0x17f0fc)[0x555c8f60fc]
../src/bootstrap-emacs(+0x17f250)[0x555c8f6250]
../src/bootstrap-emacs(+0x188048)[0x555c8ff048]
../src/bootstrap-emacs(+0x189d10)[0x555c900d10]
../src/bootstrap-emacs(+0x181330)[0x555c8f8330]
../src/bootstrap-emacs(+0x182a90)[0x555c8f9a90]
...
/bin/bash: line 1: 626 Segmentation fault EMACSLOADPATH=
'../src/bootstrap-emacs' -batch --no-site-file --no-site-lisp --eval
'(setq load-prefer-newer t)' -l comp -f
batch-byte-native-compile-for-bootstrap emacs-lisp/autoload.el
make[2]: *** [Makefile:312: emacs-lisp/autoload.elc] Error 139
make[2]: *** Waiting for unfinished jobs....
Fatal error 11: Segmentation fault
Backtrace:
../src/bootstrap-emacs(+0x1289f0)[0x555c6709f0]
../src/bootstrap-emacs(+0x46b5c)[0x555c58eb5c]
../src/bootstrap-emacs(+0x46d50)[0x555c58ed50]
../src/bootstrap-emacs(+0x12720c)[0x555c66f20c]
../src/bootstrap-emacs(+0x127298)[0x555c66f298]
linux-vdso.so.1(__kernel_rt_sigreturn+0x0)[0x7f8b597698]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc014dc)[0x7f84e154dc]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc015b0)[0x7f84e155b0]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17c890)[0x7f84390890]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17ec94)[0x7f84392c94]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17f5c8)[0x7f843935c8]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17e760)[0x7f84392760]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x1749e8)[0x7f843889e8]
/lib/aarch64-linux-gnu/libgccjit.so.0(gcc_jit_context_compile_to_file+0x94)[0x7f8437d4ec]
../src/bootstrap-emacs(+0x1b9bbc)[0x555c701bbc]
../src/bootstrap-emacs(+0x18119c)[0x555c6c919c]
../src/bootstrap-emacs(+0x1818ac)[0x555c6c98ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x1825ac)[0x555c6ca5ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x18160c)[0x555c6c960c]
../src/bootstrap-emacs(+0x180cc8)[0x555c6c8cc8]
../src/bootstrap-emacs(+0x180f7c)[0x555c6c8f7c]
../src/bootstrap-emacs(+0x181a40)[0x555c6c9a40]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x182864)[0x555c6ca864]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x1825ac)[0x555c6ca5ac]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x18160c)[0x555c6c960c]
../src/bootstrap-emacs(+0x17f0fc)[0x555c6c70fc]
../src/bootstrap-emacs(+0x1812a0)[0x555c6c92a0]
../src/bootstrap-emacs(+0x181a40)[0x555c6c9a40]
../src/bootstrap-emacs(+0x1810c4)[0x555c6c90c4]
../src/bootstrap-emacs(+0x18160c)[0x555c6c960c]
../src/bootstrap-emacs(+0x17f0fc)[0x555c6c70fc]
../src/bootstrap-emacs(+0x17f250)[0x555c6c7250]
../src/bootstrap-emacs(+0x188048)[0x555c6d0048]
../src/bootstrap-emacs(+0x189d10)[0x555c6d1d10]
../src/bootstrap-emacs(+0x181330)[0x555c6c9330]
../src/bootstrap-emacs(+0x182a90)[0x555c6caa90]
...
/bin/bash: line 1: 625 Segmentation fault EMACSLOADPATH=
'../src/bootstrap-emacs' -batch --no-site-file --no-site-lisp --eval
'(setq load-prefer-newer t)' -l comp -f
batch-byte-native-compile-for-bootstrap emacs-lisp/bytecomp.el
make[2]: *** [Makefile:312: emacs-lisp/bytecomp.elc] Error 139
Backtrace:
../src/bootstrap-emacs(+0x1289f0)[0x55647679f0]
../src/bootstrap-emacs(+0x46b5c)[0x5564685b5c]
../src/bootstrap-emacs(+0x46d50)[0x5564685d50]
../src/bootstrap-emacs(+0x12720c)[0x556476620c]
../src/bootstrap-emacs(+0x127298)[0x5564766298]
linux-vdso.so.1(__kernel_rt_sigreturn+0x0)[0x7fb85f5698]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc014dc)[0x7fb1e734dc]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0xc015b0)[0x7fb1e735b0]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17c890)[0x7fb13ee890]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17ec94)[0x7fb13f0c94]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17f5c8)[0x7fb13f15c8]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x17e760)[0x7fb13f0760]
/lib/aarch64-linux-gnu/libgccjit.so.0(+0x1749e8)[0x7fb13e69e8]
/lib/aarch64-linux-gnu/libgccjit.so.0(gcc_jit_context_compile_to_file+0x94)[0x7fb13db4ec]
../src/bootstrap-emacs(+0x1b9bbc)[0x55647f8bbc]
../src/bootstrap-emacs(+0x18119c)[0x55647c019c]
../src/bootstrap-emacs(+0x1818ac)[0x55647c08ac]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x1825ac)[0x55647c15ac]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x18160c)[0x55647c060c]
../src/bootstrap-emacs(+0x180cc8)[0x55647bfcc8]
../src/bootstrap-emacs(+0x180f7c)[0x55647bff7c]
../src/bootstrap-emacs(+0x181a40)[0x55647c0a40]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x182864)[0x55647c1864]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x1825ac)[0x55647c15ac]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x18160c)[0x55647c060c]
../src/bootstrap-emacs(+0x17f0fc)[0x55647be0fc]
../src/bootstrap-emacs(+0x1812a0)[0x55647c02a0]
../src/bootstrap-emacs(+0x181a40)[0x55647c0a40]
../src/bootstrap-emacs(+0x1810c4)[0x55647c00c4]
../src/bootstrap-emacs(+0x18160c)[0x55647c060c]
../src/bootstrap-emacs(+0x17f0fc)[0x55647be0fc]
../src/bootstrap-emacs(+0x17f250)[0x55647be250]
../src/bootstrap-emacs(+0x188048)[0x55647c7048]
../src/bootstrap-emacs(+0x189d10)[0x55647c8d10]
../src/bootstrap-emacs(+0x181330)[0x55647c0330]
../src/bootstrap-emacs(+0x182a90)[0x55647c1a90]
...
/bin/bash: line 1: 624 Segmentation fault EMACSLOADPATH=
'../src/bootstrap-emacs' -batch --no-site-file --no-site-lisp --eval
'(setq load-prefer-newer t)' -l comp -f
batch-byte-native-compile-for-bootstrap emacs-lisp/comp.el
make[2]: *** [Makefile:312: emacs-lisp/comp.elc] Error 139
make[2]: Leaving directory '/home/pi/Downloads/emacs/master/lisp'
make[1]: *** [Makefile:827: bootstrap-emacs.pdmp] Error 2
make[1]: Leaving directory '/home/pi/Downloads/emacs/master/src'
make: *** [Makefile:424: src] Error 2
----
On Thu, Mar 5, 2020 at 4:55 AM Adam Porter <adam@alphapapa.net> wrote:
>
> Hi Andrea,
>
> Andrea Corallo <akrl@sdf.org> writes:
>
> > - Finally I've set up a docker image for people willing to test it
> > without having to configure and compile Emacs and libgccjit. Now that
> > I've learned how it works I plan to use it to setup a small CI to keep
> > an eye that the latest GCC does not break with our build.
>
> Thank you for setting up the Docker image. I had tried to build the
> native-comp branch on my system but finally was unable to overcome a
> weird error related to libgccjit (probably because I need to upgrade my
> system). I even tried using Guix to build it, but libgccjit isn't
> packaged in it (and I'm completely new to Guix packaging, so I wasn't
> able to figure out how to adjust the GCC build for that in a reasonable
> amount of time).
>
> I managed to get the Docker image working, and I did a little testing
> with one of my packages, org-ql, which is for searching Org files. I
> installed it and its dependencies, then used native-comp-async to
> compile all of ~/.emacs.d/elpa. Then I restarted Emacs to ensure that
> the native libraries were loaded, and finally I ran the following code
> and observed a more than 2x improvement!
>
> #+BEGIN_SRC elisp
> ;; Reset the cache to force search to run again.
> (setf org-ql-cache (make-hash-table :test #'equal))
>
> ;; Set my personal groups (omitted) and enable org-super-agenda-mode
> ;; to display in groups.
> (setf org-super-agenda-groups my-groups)
> (org-super-agenda-mode)
>
> ;; Load all my Org files and search them.
> (let* ((enable-local-variables nil)
> (files (directory-files org-directory 'full (rx ".org" eos))))
> (dolist (file files)
> (find-file-noselect file 'nowarn))
> (benchmark-run-compiled
> (org-ql-search files
> ;; The (tags) selector tends to be slow compared to other
> ;; predicates because of searching the hierarchy repeatedly
> ;; and tag inheritance, so it makes for an interesting test.
> '(and (todo) (tags "Emacs"))
> :super-groups org-super-agenda-groups)))
> #+END_SRC
>
> #+RESULTS:
> | 5.819892666 | 1 | 0.6101534569999956 | In Emacs 26.3
> | 2.317804428 | 3 | 0.6223487619999997 | In Emacs 28 with native-comp
>
> Then I tested this more complex query and observed an even more
> impressive speedup of about 3.4x!
>
> '(and (not (done))
> (or (habit)
> (deadline auto)
> (scheduled :to today)
> (ts-active :on today)))
>
> #+RESULTS:
> | 8.377843199 | 1 | 0.6277314330000081 | In Emacs 26.3
> | 2.436048375 | 2 | 0.4278436059999997 | In Emacs 28 with native-comp
>
> This is very exciting! The difference between 8.4 seconds and 2.4
> seconds is a huge usability improvement.
>
> And everything seems to work fine! No bugs, errors, or crashes of any
> kind.
>
> I did encounter a minor issue related to macro-expansion and loading
> that may be my fault. When I tried to (require 'org-ql) or call a
> function that is autoloaded from it, I got an error saying that the .eln
> file did not provide the org-ql feature. I surmised that meant that the
> file hadn't been compiled properly, so I looked at the async compilation
> log and saw an error saying that the variable org-ql-predicates was not
> defined. At this URL you can see a macro definition,
> org-ql--def-plain-query-fn, and the call to it that's wrapped in
> cl-eval-when to ensure it works properly with the byte-compiler:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L1671
>
> The macro uses the value of org-ql-predicates, which is defined here:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L126
>
> But when the org-ql--defpred macro:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L140
>
> ...is called, it adds to that variable. So all of that works with the
> byte-compiler, but apparently not with the native compilation.
>
> To work around it, I wrapped (defvar org-ql-predicates...) in
> eval-and-compile, and then wrapped all of the calls to org-ql--defpred
> in eval-and-compile as well. Then I deleted the org-ql.eln file and
> recompiled it, restarted Emacs, and then everything worked fine.
>
> I'm guessing that this issue is "native" to the native-compilation and
> might need to be documented in some way, but I wouldn't expect it to
> affect many packages, only ones that do tricky things with variables and
> macro-expansion like this.
>
> So, a few ideas for fixes or improvements:
>
> 1. I guess the .eln file should not have been produced or left behind
> in the directory, since there was a compilation error.
> 2. The compilation error should probably be shown as a warning or
> something, because it's easily lost in the async compilation log
> buffer.
> 3. It would probably be good if the loader fell back to .elc and .el
> files if the .eln file doesn't seem to work, and showed a warning
> when doing so. The library would remain usable in its byte-compiled
> form, and in the case of a package like this, the user could then
> report the native-compilation error to the developer.
>
> Other than that minor issue, everything works and feels very fast! This
> is great!
>
> > ~eln~ files are compiled in specific sub-folders taking in account
> > host architecture and current Emacs configuration to disambiguate
> > possible incompatibilities.
>
> > As example a file ~.../foo.el~ will compile into something like
> > ~.../x86_64-pc-linux-gnu-12383a81d4f33a87/foo.eln~.
>
> FYI, I did not see the .eln files being put in subdirectories like that
> in the Docker image. Maybe it didn't receive that update yet.
>
> This is all very exciting. I'm imagining an update to package.el that
> would byte-compile packages immediately, as usual, then start
> native-compile-async'ing them in the background, loading each file's
> native version as it completes, with the packages being immediately
> usable and seeming to get faster as each file is compiled and loaded.
>
> > I'm generally very satisfied (surprised) about the stability of the
> > toy. I'm looking forward going into compile time mitigation. I guess
> > I'll start this weekend with deferred compilation and GCC profiling.
>
> I think it isn't a toy anymore! :) Thanks very much for your work on
> this. Exciting times for Emacs, as this feature, in one form or
> another, has been eagerly awaited for years.
>
>
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
@ 2020-03-05 13:27 ` Andrea Corallo
2020-03-08 14:37 ` Gong-Yi Liao 廖宮毅
2020-03-15 19:00 ` Gong-Yi Liao 廖宮毅
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-05 13:27 UTC (permalink / raw)
To: Gong-Yi Liao 廖宮毅; +Cc: Adam Porter, emacs-devel
Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
> That's pretty good on (I assume) x86_64, but native-comp branch seems
> not working on ARM64 Debian Buster at this moment:
> Not sure if it's a GCCJIT issue or the native-comp branch issue:
>
> Any comments on this compilation log are appreciated.
Hi Gong-Yi,
I guess this is a problem of the libgccjit package shipped with the
distro. I suggest you to recompile a recent version of it. You can
also take a look as example to the docker image build process if you
like:
https://gitlab.com/koral/emacs-nativecomp-dockerfile/-/blob/master/Dockerfile
I use aarch64 machines regularly so I guess you should solve the issue
just with that.
Regards
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 13:27 ` Andrea Corallo
@ 2020-03-08 14:37 ` Gong-Yi Liao 廖宮毅
2020-03-08 15:13 ` Andrea Corallo
2020-03-15 19:00 ` Gong-Yi Liao 廖宮毅
1 sibling, 1 reply; 210+ messages in thread
From: Gong-Yi Liao 廖宮毅 @ 2020-03-08 14:37 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Adam Porter, emacs-devel
Hi Andrea,
I tried to cross compile gccjit and emacs for aarch64 target on a
x86-64 machine as the raspberry pi 4b is not powerful enough for this
type of tasks but I haven't get anything fruitful so far.
When I complied these two packages on a raspberry pi 4b, compiling
gccjit was ok, executable binaries were eventually generated.
But emacs/native-comp always fail when native compiling leim related elc/eln.
Do you have any recommendation for setting up a cross compilation
environment for that?
Thanks,
Gong-Yi.
On Thu, Mar 5, 2020 at 7:27 AM Andrea Corallo <akrl@sdf.org> wrote:
>
> Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
>
> > That's pretty good on (I assume) x86_64, but native-comp branch seems
> > not working on ARM64 Debian Buster at this moment:
> > Not sure if it's a GCCJIT issue or the native-comp branch issue:
> >
> > Any comments on this compilation log are appreciated.
>
> Hi Gong-Yi,
>
> I guess this is a problem of the libgccjit package shipped with the
> distro. I suggest you to recompile a recent version of it. You can
> also take a look as example to the docker image build process if you
> like:
>
> https://gitlab.com/koral/emacs-nativecomp-dockerfile/-/blob/master/Dockerfile
>
> I use aarch64 machines regularly so I guess you should solve the issue
> just with that.
>
> Regards
>
> Andrea
>
> --
> akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-08 14:37 ` Gong-Yi Liao 廖宮毅
@ 2020-03-08 15:13 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-08 15:13 UTC (permalink / raw)
To: Gong-Yi Liao 廖宮毅; +Cc: Adam Porter, emacs-devel
Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
> Hi Andrea,
>
> I tried to cross compile gccjit and emacs for aarch64 target on a
> x86-64 machine as the raspberry pi 4b is not powerful enough for this
> type of tasks but I haven't get anything fruitful so far.
>
> When I complied these two packages on a raspberry pi 4b, compiling
> gccjit was ok, executable binaries were eventually generated.
>
> But emacs/native-comp always fail when native compiling leim related elc/eln.
>
> Do you have any recommendation for setting up a cross compilation
> environment for that?
>
> Thanks,
> Gong-Yi.
Hi Gong-Yi,
I'm not sure a libgccjit cross compilation setup was ever attempted
before. Not even sure about cross compiling the standard Emacs but as
of today I think cross compiling native-comp would be pure
experimentation/development.
If you have a working libgccjit on the pi I suggest you to start with
modifying the Emacs make command as follow:
make bootstrap BYTE_COMPILE_EXTRA_FLAGS='--eval "(setq comp-speed 0)"'
speed zero is one magnitude order faster in compiling being still faster
then the bytecompiler in the resulting code you'll execute. At least
you can start testing the rest of the system. Once is boostrapped
you'll be able to compile pieces of elisp at speed 2 if you like.
Hope it helps.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 13:27 ` Andrea Corallo
2020-03-08 14:37 ` Gong-Yi Liao 廖宮毅
@ 2020-03-15 19:00 ` Gong-Yi Liao 廖宮毅
2020-03-15 20:00 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Gong-Yi Liao 廖宮毅 @ 2020-03-15 19:00 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Adam Porter, emacs-devel
I have built from the current (2020-3-15 13:55 CDT) native-comp branch
with the following setting:
1. install prefix: /home/gong-yi/.local/emacs/native-comp
2.. built with gcc 8.3 on Debian buster 10
3. The git clone to /home/gong-yi/Downloads/emacs/master
And I just noticed that the resulted built seems not build-location-independent:
gong-yi@buster1:~/Downloads/emacs$ mv master temp
gong-yi@buster1:~/Downloads/emacs$ cd
gong-yi@buster1:~$ cd .local/emacs/native-comp/bin/
gong-yi@buster1:~/.local/emacs/native-comp/bin$ ./emacs
emacs: /home/gong-yi/Downloads/emacs/master/lisp/eln-x86_64-pc-linux-gnu-4f8c8dbc445c9e9e/custom.eln:
cannot open shared object file: No such file or directory
It suggests eln files' location/path are locked regardless where the
installation path is.
Any solution?
Thanks,
Gong-Yi.
On Thu, Mar 5, 2020 at 7:27 AM Andrea Corallo <akrl@sdf.org> wrote:
>
> Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
>
> > That's pretty good on (I assume) x86_64, but native-comp branch seems
> > not working on ARM64 Debian Buster at this moment:
> > Not sure if it's a GCCJIT issue or the native-comp branch issue:
> >
> > Any comments on this compilation log are appreciated.
>
> Hi Gong-Yi,
>
> I guess this is a problem of the libgccjit package shipped with the
> distro. I suggest you to recompile a recent version of it. You can
> also take a look as example to the docker image build process if you
> like:
>
> https://gitlab.com/koral/emacs-nativecomp-dockerfile/-/blob/master/Dockerfile
>
> I use aarch64 machines regularly so I guess you should solve the issue
> just with that.
>
> Regards
>
> Andrea
>
> --
> akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-15 19:00 ` Gong-Yi Liao 廖宮毅
@ 2020-03-15 20:00 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-15 20:00 UTC (permalink / raw)
To: Gong-Yi Liao 廖宮毅; +Cc: Adam Porter, emacs-devel
Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
> I have built from the current (2020-3-15 13:55 CDT) native-comp branch
> with the following setting:
>
> 1. install prefix: /home/gong-yi/.local/emacs/native-comp
> 2.. built with gcc 8.3 on Debian buster 10
> 3. The git clone to /home/gong-yi/Downloads/emacs/master
>
> And I just noticed that the resulted built seems not build-location-independent:
>
> gong-yi@buster1:~/Downloads/emacs$ mv master temp
> gong-yi@buster1:~/Downloads/emacs$ cd
> gong-yi@buster1:~$ cd .local/emacs/native-comp/bin/
> gong-yi@buster1:~/.local/emacs/native-comp/bin$ ./emacs
> emacs: /home/gong-yi/Downloads/emacs/master/lisp/eln-x86_64-pc-linux-gnu-4f8c8dbc445c9e9e/custom.eln:
> cannot open shared object file: No such file or directory
>
> It suggests eln files' location/path are locked regardless where the
> installation path is.
>
> Any solution?
> Thanks,
> Gong-Yi.
Hi Gong-Yi Liao,
yes this is expected and to be fixed :)
Now absolute paths are used for the dump and reload mechanism and we
have to change these into relatives.
Because of that also make install can't work.
Thanks for feedbacking!
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 10:54 ` Adam Porter
2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
@ 2020-03-05 13:19 ` Andrea Corallo
2020-03-05 14:42 ` Adam Porter
2020-03-06 21:15 ` Adam Porter
2020-03-11 22:04 ` Andrea Corallo
2 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-05 13:19 UTC (permalink / raw)
To: Adam Porter; +Cc: emacs-devel
Adam Porter <adam@alphapapa.net> writes:
>
> #+RESULTS:
> | 5.819892666 | 1 | 0.6101534569999956 | In Emacs 26.3
> | 2.317804428 | 3 | 0.6223487619999997 | In Emacs 28 with native-comp
>
> Then I tested this more complex query and observed an even more
> impressive speedup of about 3.4x!
>
> '(and (not (done))
> (or (habit)
> (deadline auto)
> (scheduled :to today)
> (ts-active :on today)))
>
> #+RESULTS:
> | 8.377843199 | 1 | 0.6277314330000081 | In Emacs 26.3
> | 2.436048375 | 2 | 0.4278436059999997 | In Emacs 28 with native-comp
>
> This is very exciting! The difference between 8.4 seconds and 2.4
> seconds is a huge usability improvement.
Wow these are very good results, it's interesting to collect feedback
from real world scenarios, thanks.
> I did encounter a minor issue related to macro-expansion and loading
> that may be my fault. When I tried to (require 'org-ql) or call a
> function that is autoloaded from it, I got an error saying that the .eln
> file did not provide the org-ql feature. I surmised that meant that the
> file hadn't been compiled properly, so I looked at the async compilation
> log and saw an error saying that the variable org-ql-predicates was not
> defined. At this URL you can see a macro definition,
> org-ql--def-plain-query-fn, and the call to it that's wrapped in
> cl-eval-when to ensure it works properly with the byte-compiler:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L1671
>
> The macro uses the value of org-ql-predicates, which is defined here:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L126
>
> But when the org-ql--defpred macro:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L140
>
> ...is called, it adds to that variable. So all of that works with the
> byte-compiler, but apparently not with the native compilation.
>
> To work around it, I wrapped (defvar org-ql-predicates...) in
> eval-and-compile, and then wrapped all of the calls to org-ql--defpred
> in eval-and-compile as well. Then I deleted the org-ql.eln file and
> recompiled it, restarted Emacs, and then everything worked fine.
>
> I'm guessing that this issue is "native" to the native-compilation and
> might need to be documented in some way, but I wouldn't expect it to
> affect many packages, only ones that do tricky things with variables and
> macro-expansion like this.
This is strange. The semantic of the native compiled code should be
exactly the same of the byte-compiled one. I may look at this in the
weekend if I manage to. Thanks for reporting.
> So, a few ideas for fixes or improvements:
>
> 1. I guess the .eln file should not have been produced or left behind
> in the directory, since there was a compilation error.
Agree, in case of error no eln should have been produced. I thought it
was the case but if it's not that's a bug.
> 2. The compilation error should probably be shown as a warning or
> something, because it's easily lost in the async compilation log
> buffer.
Are you suggesting to use the minibuffer to report these?
> 3. It would probably be good if the loader fell back to .elc and .el
> files if the .eln file doesn't seem to work, and showed a warning
> when doing so. The library would remain usable in its byte-compiled
> form, and in the case of a package like this, the user could then
> report the native-compilation error to the developer.
>
> Other than that minor issue, everything works and feels very fast! This
> is great!
>
>> ~eln~ files are compiled in specific sub-folders taking in account
>> host architecture and current Emacs configuration to disambiguate
>> possible incompatibilities.
>
>> As example a file ~.../foo.el~ will compile into something like
>> ~.../x86_64-pc-linux-gnu-12383a81d4f33a87/foo.eln~.
>
> FYI, I did not see the .eln files being put in subdirectories like that
> in the Docker image. Maybe it didn't receive that update yet.
Ooops, should be ok by now.
> This is all very exciting. I'm imagining an update to package.el that
> would byte-compile packages immediately, as usual, then start
> native-compile-async'ing them in the background, loading each file's
> native version as it completes, with the packages being immediately
> usable and seeming to get faster as each file is compiled and loaded.
Yes that's the idea but I'd like to make it not package.el specific but
for every .elc being loaded in Emacs.
https://lists.gnu.org/archive/html/emacs-devel/2020-02/msg00225.html
Thanks for your mail is great to have feedback to work on.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 13:19 ` Andrea Corallo
@ 2020-03-05 14:42 ` Adam Porter
2020-03-05 19:46 ` Andrea Corallo
2020-03-06 21:15 ` Adam Porter
1 sibling, 1 reply; 210+ messages in thread
From: Adam Porter @ 2020-03-05 14:42 UTC (permalink / raw)
To: emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
>> 2. The compilation error should probably be shown as a warning or
>> something, because it's easily lost in the async compilation log
>> buffer.
>
> Are you suggesting to use the minibuffer to report these?
I was thinking an actual warning, i.e. with display-warning. Maybe it
could be controlled with an option, so at least testers and package
authors could be sure to notice them rather than having them get lost
among all the other messages. :)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 14:42 ` Adam Porter
@ 2020-03-05 19:46 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-05 19:46 UTC (permalink / raw)
To: Adam Porter; +Cc: emacs-devel
Adam Porter <adam@alphapapa.net> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>>> 2. The compilation error should probably be shown as a warning or
>>> something, because it's easily lost in the async compilation log
>>> buffer.
>>
>> Are you suggesting to use the minibuffer to report these?
>
> I was thinking an actual warning, i.e. with display-warning. Maybe it
> could be controlled with an option, so at least testers and package
> authors could be sure to notice them rather than having them get lost
> among all the other messages. :)
Sounds like a good suggestion to me thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 13:19 ` Andrea Corallo
2020-03-05 14:42 ` Adam Porter
@ 2020-03-06 21:15 ` Adam Porter
2020-03-06 21:23 ` Adam Porter
1 sibling, 1 reply; 210+ messages in thread
From: Adam Porter @ 2020-03-06 21:15 UTC (permalink / raw)
To: emacs-devel
Hi Andrea,
Andrea Corallo <akrl@sdf.org> writes:
>> To work around it, I wrapped (defvar org-ql-predicates...) in
>> eval-and-compile, and then wrapped all of the calls to org-ql--defpred
>> in eval-and-compile as well. Then I deleted the org-ql.eln file and
>> recompiled it, restarted Emacs, and then everything worked fine.
>>
>> I'm guessing that this issue is "native" to the native-compilation and
>> might need to be documented in some way, but I wouldn't expect it to
>> affect many packages, only ones that do tricky things with variables and
>> macro-expansion like this.
>
> This is strange. The semantic of the native compiled code should be
> exactly the same of the byte-compiled one. I may look at this in the
> weekend if I manage to. Thanks for reporting.
FYI, I pulled the new Docker image, the one that does create the
arch-specific directories for the ELN files, then I installed org-ql
again, without those modifications, and I didn't get that error again,
and it seems to be working fine. So hopefully it was either a fluke or
something fixed by your latest changes.
One small issue I noticed: While working on another package, I caused a
segfault when calling edebug-defun. I restarted Emacs and wasn't able
to reproduce it, and I continued using Edebug on that and other code
without any crashes. I also noticed some other unusual behavior with
Edebug while stepping through an Edebugged function (pressing "g" did
not skip to the next breakpoint, instead it seemed to behave like "n"),
and I doubt that has anything to do with the native-comp branch, so
maybe the segfault doesn't either.
Thanks again for all your work on this.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-06 21:15 ` Adam Porter
@ 2020-03-06 21:23 ` Adam Porter
2020-03-06 21:52 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Adam Porter @ 2020-03-06 21:23 UTC (permalink / raw)
To: emacs-devel
Hi again Andrea,
Adam Porter <adam@alphapapa.net> writes:
>>> To work around it, I wrapped (defvar org-ql-predicates...) in
>>> eval-and-compile, and then wrapped all of the calls to org-ql--defpred
>>> in eval-and-compile as well. Then I deleted the org-ql.eln file and
>>> recompiled it, restarted Emacs, and then everything worked fine.
>>>
>>> I'm guessing that this issue is "native" to the native-compilation and
>>> might need to be documented in some way, but I wouldn't expect it to
>>> affect many packages, only ones that do tricky things with variables and
>>> macro-expansion like this.
>>
>> This is strange. The semantic of the native compiled code should be
>> exactly the same of the byte-compiled one. I may look at this in the
>> weekend if I manage to. Thanks for reporting.
>
> FYI, I pulled the new Docker image, the one that does create the
> arch-specific directories for the ELN files, then I installed org-ql
> again, without those modifications, and I didn't get that error again,
> and it seems to be working fine. So hopefully it was either a fluke or
> something fixed by your latest changes.
Please disregard that. I had accidentally recompiled the modified
version with the fixes I mentioned. When I recompiled the version
without those fixes, the error returned.
So I was trying to save you the trouble of looking into an false error
report, but in the process I made a false false error report. If this
gets any more confusing, I'll write my reports in Lisp instead. :)
I'll continue testing with more packages native-compiled.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-06 21:23 ` Adam Porter
@ 2020-03-06 21:52 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-06 21:52 UTC (permalink / raw)
To: Adam Porter; +Cc: emacs-devel
Adam Porter <adam@alphapapa.net> writes:
> Hi again Andrea,
>
> Adam Porter <adam@alphapapa.net> writes:
>
>>>> To work around it, I wrapped (defvar org-ql-predicates...) in
>>>> eval-and-compile, and then wrapped all of the calls to org-ql--defpred
>>>> in eval-and-compile as well. Then I deleted the org-ql.eln file and
>>>> recompiled it, restarted Emacs, and then everything worked fine.
>>>>
>>>> I'm guessing that this issue is "native" to the native-compilation and
>>>> might need to be documented in some way, but I wouldn't expect it to
>>>> affect many packages, only ones that do tricky things with variables and
>>>> macro-expansion like this.
>>>
>>> This is strange. The semantic of the native compiled code should be
>>> exactly the same of the byte-compiled one. I may look at this in the
>>> weekend if I manage to. Thanks for reporting.
>>
>> FYI, I pulled the new Docker image, the one that does create the
>> arch-specific directories for the ELN files, then I installed org-ql
>> again, without those modifications, and I didn't get that error again,
>> and it seems to be working fine. So hopefully it was either a fluke or
>> something fixed by your latest changes.
>
> Please disregard that. I had accidentally recompiled the modified
> version with the fixes I mentioned. When I recompiled the version
> without those fixes, the error returned.
This is very good news. It means will be easy to reproduce :)
> So I was trying to save you the trouble of looking into an false error
> report, but in the process I made a false false error report. If this
> gets any more confusing, I'll write my reports in Lisp instead. :)
>
> I'll continue testing with more packages native-compiled.
Thanks!
Also for the future we can use the issue tracker to track these down.
https://lists.gnu.org/archive/html/emacs-devel/2020-03/msg00146.html
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-05 10:54 ` Adam Porter
2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
2020-03-05 13:19 ` Andrea Corallo
@ 2020-03-11 22:04 ` Andrea Corallo
2020-03-12 17:10 ` Adam Porter
2020-03-12 17:34 ` Stefan Monnier
2 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-11 22:04 UTC (permalink / raw)
To: Adam Porter; +Cc: Stefan Monnier, emacs-devel
Adam Porter <adam@alphapapa.net> writes:
> Hi Andrea,
Hi Adam,
> I did encounter a minor issue related to macro-expansion and loading
> that may be my fault. When I tried to (require 'org-ql) or call a
> function that is autoloaded from it, I got an error saying that the .eln
> file did not provide the org-ql feature. I surmised that meant that the
> file hadn't been compiled properly, so I looked at the async compilation
> log and saw an error saying that the variable org-ql-predicates was not
> defined. At this URL you can see a macro definition,
> org-ql--def-plain-query-fn, and the call to it that's wrapped in
> cl-eval-when to ensure it works properly with the byte-compiler:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L1671
>
> The macro uses the value of org-ql-predicates, which is defined here:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L126
>
> But when the org-ql--defpred macro:
>
> https://github.com/alphapapa/org-ql/blob/06481960764a55a36d886e1db79a58258d5d2ffb/org-ql.el#L140
>
> ...is called, it adds to that variable. So all of that works with the
> byte-compiler, but apparently not with the native compilation.
I took a look to the issue. I think issue reduces to:
#+BEGIN_SRC lisp
(defvar repro-xxx 123)
(cl-eval-when (compile load eval)
(defmacro repro-macro ()
"foo"
repro-xxx)
(repro-macro))
(provide 'repro)
#+END_SRC
And this can be further reduced to:
#+BEGIN_SRC lisp
(defvar repro-xxx)
(cl-eval-when (compile)
repro-xxx)
(provide 'repro)
#+END_SRC
Both of these do *not* byte-compile. I'm not an expert but to me it
makes sense that `repro-xxx' is not defined in the compilation
enviroment and I think the byte-compiler is very much correct in
signaling the error.
The reason why your code is byte-compiled correctly during the package
installation is that apparently package.el is loading the non compiled
version *before* running the byte-compiler through
`package--load-files-for-activation'. As a consequence when the
byte-compiler runs the defvar already took effect.
I'm not sure this is what we want in general because it can mask issues
but I'm new to Emacs development so there's very possibly a reason to
that. Perhaps someone will comment this.
I think your fix to wrap the defvar is correct.
That said we have to fix the native compiler that not only is not
signaling in a sufficiently visible way the error, but is also producing
an eln just ignoring it :)
I'll look into this second point as next.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-11 22:04 ` Andrea Corallo
@ 2020-03-12 17:10 ` Adam Porter
2020-03-12 17:34 ` Stefan Monnier
1 sibling, 0 replies; 210+ messages in thread
From: Adam Porter @ 2020-03-12 17:10 UTC (permalink / raw)
To: emacs-devel
Hi Andrea,
On Wed, Mar 11, 2020 at 5:04 PM Andrea Corallo <akrl@sdf.org> wrote:
> The reason why your code is byte-compiled correctly during the package
> installation is that apparently package.el is loading the non compiled
> version *before* running the byte-compiler through
> `package--load-files-for-activation'. As a consequence when the
> byte-compiler runs the defvar already took effect.
>
> I think your fix to wrap the defvar is correct.
>
> That said we have to fix the native compiler that not only is not
> signaling in a sufficiently visible way the error, but is also producing
> an eln just ignoring it :)
>
> I'll look into this second point as next.
Thanks very much for digging into that. Glad to know that I was
indeed doing something at least somewhat wrong, and how to fix it.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-11 22:04 ` Andrea Corallo
2020-03-12 17:10 ` Adam Porter
@ 2020-03-12 17:34 ` Stefan Monnier
2020-03-12 18:03 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-03-12 17:34 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Adam Porter, emacs-devel
> The reason why your code is byte-compiled correctly during the package
> installation is that apparently package.el is loading the non compiled
> version *before* running the byte-compiler through
> `package--load-files-for-activation'.
Package.el doesn't do that directly. It may be the indirect result of
something package.el does, but the only thing that package.el does which
seems relevant is to load the `<pkg>-autoloads.el` file before starting
the compilation.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-12 17:34 ` Stefan Monnier
@ 2020-03-12 18:03 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-12 18:03 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Adam Porter, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>> The reason why your code is byte-compiled correctly during the package
>> installation is that apparently package.el is loading the non compiled
>> version *before* running the byte-compiler through
>> `package--load-files-for-activation'.
>
> Package.el doesn't do that directly. It may be the indirect result of
> something package.el does, but the only thing that package.el does which
> seems relevant is to load the `<pkg>-autoloads.el` file before starting
> the compilation.
>
Hi Stefan,
This is something I've seen yesterday debugging and the practical
experience that Adam's package gets byte-compiled confirms that. If you
think is important to look into it I can have a look.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update7
2020-03-04 19:34 ` On elisp running native Andrea Corallo
2020-03-04 20:21 ` Michael Albinus
2020-03-05 10:54 ` Adam Porter
@ 2020-03-23 21:36 ` Andrea Corallo
2020-03-24 2:59 ` Gong-Yi Liao 廖宮毅
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
2 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-23 21:36 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier
Hi all,
This is to update on what has been going on on the feature/native-comp
branch.
I've written a small summary that touches: asyncronous compilation,
deferred compilation, fast bootstrap and some more under "Update 7" on:
https://akrl.sdf.org/gccemacs.html
Please feedback comments opinions and or questions.
Regards
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update7
2020-03-23 21:36 ` On elisp running native - Update7 Andrea Corallo
@ 2020-03-24 2:59 ` Gong-Yi Liao 廖宮毅
2020-03-24 8:44 ` Andrea Corallo
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Gong-Yi Liao 廖宮毅 @ 2020-03-24 2:59 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
On my machine (AMD A10-6800K), time spent on compiling current
native-comp branch now reduce to 35 minutes compared to 5 hours in
previous attempts.
With comp-deferred-compilation turned on, I got the following error
messages while native-compiling SLIME package, not sure if the comp.el
provides the type checking:
----------------------------
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-xref-browser.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-typeout-frame.el...
Wrong tyWrong type argument: lispe argument: listp, #<subr slime-alistify>
tp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-tramp.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-trace-dialog.el...
Wrong type argument: listp, #<subr slimWrong type argument: listp,
#<subr se-alistify>
lime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-sprof.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-snapshot.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-scratch.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-scheme.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-sbcl-exts.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-repl.el...
Wrong type argument: listp, #<subr slime-alistWrong type argument:
listp, #<subr slime-alistify>
ify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-references.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-quicklisp.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-presentations.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-presentation-streams.el...
Wrong type argument: listp, #<suWrong type argument: listp, #<subr
slime-alistify>
br slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-parse.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-package-fu.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-mrepl.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-media.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-mdot-fu.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-macrostep.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-listener-hooks.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-indentation.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-hyperdoc.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-highlight-edits.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-fuzzy.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-fontifying-fu.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-fancy.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-fancy-trace.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-fancy-inspector.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-enclosing-context.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-editing-commands.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-compiler-notes-tree.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-clipboard.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-cl-indent.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-c-p-c.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-buffer-streams.el...
Wrong type argument: listp, #<subr slime-Wrong type argument: listp,
#<subr slime-alalistify>
istify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-banner.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-autodoc.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/slime-asdf.el...
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/inferior-slime.el...
Wrong type argument: listp, #<subr slime-alistify>
Wrong type argument: listp, #<subr slime-alistify>
Compiling /home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/bridge.el...
In toplevel form:
bridge.el:69:13: Warning: Package cl is deprecated
can’t native compile a non lexical scoped function:
"/home/gong-yi/.emacs.d-28/elpa/slime-20200319.1939/contrib/bridge.el",
hand-bridge
Compilation finished.
--------------------
On Mon, Mar 23, 2020 at 4:37 PM Andrea Corallo <akrl@sdf.org> wrote:
>
> Hi all,
>
> This is to update on what has been going on on the feature/native-comp
> branch.
>
> I've written a small summary that touches: asyncronous compilation,
> deferred compilation, fast bootstrap and some more under "Update 7" on:
>
> https://akrl.sdf.org/gccemacs.html
>
> Please feedback comments opinions and or questions.
>
> Regards
>
> Andrea
>
> --
> akrl@sdf.org
>
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update7
2020-03-24 2:59 ` Gong-Yi Liao 廖宮毅
@ 2020-03-24 8:44 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-24 8:44 UTC (permalink / raw)
To: Gong-Yi Liao 廖宮毅
Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier, emacs-devel
Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
> With comp-deferred-compilation turned on, I got the following error
> messages while native-compiling SLIME package, not sure if the comp.el
> provides the type checking:
>
Hi Gong-Yi,
Please just double check you are on the latest feature/native-comp
73ced8c23e (I ask that becasue 855940df6b solved a bug with the exact
same symptoms).
In case you are are please open a bug on the bug tracker so we can treat
it as such.
Thanks!
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 8
2020-03-23 21:36 ` On elisp running native - Update7 Andrea Corallo
2020-03-24 2:59 ` Gong-Yi Liao 廖宮毅
@ 2020-05-16 15:24 ` Andrea Corallo
2020-05-16 16:21 ` Eli Zaretskii
` (2 more replies)
1 sibling, 3 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-05-16 15:24 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Paul Eggert, Stefan Monnier
Hi all,
I thought was time to write an update on my dev blog about the status of
the native compiler branch.
https://akrl.sdf.org/gccemacs.html
Please follow up with comments, suggestions on missing features or
observations so we can prioritize correctly to work in the direction of
making it mergeable. I value any comment or maintainer's indication on
that.
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 8
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
@ 2020-05-16 16:21 ` Eli Zaretskii
2020-05-16 20:44 ` Andrea Corallo
2020-05-16 18:22 ` Gregor Zattler
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-05-16 16:21 UTC (permalink / raw)
To: Andrea Corallo; +Cc: eggert, monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Sat, 16 May 2020 15:24:01 +0000
> Cc: Eli Zaretskii <eliz@gnu.org>, Paul Eggert <eggert@cs.ucla.edu>,
> Stefan Monnier <monnier@iro.umontreal.ca>
>
> https://akrl.sdf.org/gccemacs.html
>
> Please follow up with comments, suggestions on missing features or
> observations so we can prioritize correctly to work in the direction of
> making it mergeable. I value any comment or maintainer's indication on
> that.
FWIW, my impression is that relatively serious problems are still
being reported quite frequently, including crashes.
I also think we should figure out the more high-level problems I
brought up in the "bird-eye view" thread, because deployment
considerations do matter.
Thanks.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 8
2020-05-16 16:21 ` Eli Zaretskii
@ 2020-05-16 20:44 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-05-16 20:44 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: eggert, monnier, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Date: Sat, 16 May 2020 15:24:01 +0000
>> Cc: Eli Zaretskii <eliz@gnu.org>, Paul Eggert <eggert@cs.ucla.edu>,
>> Stefan Monnier <monnier@iro.umontreal.ca>
>>
>> https://akrl.sdf.org/gccemacs.html
>>
>> Please follow up with comments, suggestions on missing features or
>> observations so we can prioritize correctly to work in the direction of
>> making it mergeable. I value any comment or maintainer's indication on
>> that.
>
> FWIW, my impression is that relatively serious problems are still
> being reported quite frequently, including crashes.
>
> I also think we should figure out the more high-level problems I
> brought up in the "bird-eye view" thread, because deployment
> considerations do matter.
>
> Thanks.
Hi Eli,
far from me saying that the branch is completed or finished.
IOW given the branch is under serious development if you can keep an eye
(or more) on what is going on with that I think will be extremely useful
to lead it fruition. I see you are doing that now so I thank you for
that.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 8
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
2020-05-16 16:21 ` Eli Zaretskii
@ 2020-05-16 18:22 ` Gregor Zattler
2020-05-16 19:59 ` Andrea Corallo
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
2 siblings, 1 reply; 210+ messages in thread
From: Gregor Zattler @ 2020-05-16 18:22 UTC (permalink / raw)
To: emacs-devel
Hi Andrea, emacs developers,
* Andrea Corallo <akrl@sdf.org> [2020-05-16; 15:24]:
> Please follow up with comments, suggestions on missing features or
> observations
I tried to build feature/native-comp as of today (commit
d6f6353cfdbbea5501915675081265b4dc4591e3) on an up-to-date
debian buster (10.4) with version 8.3.0-6 of libgccjit
installed and got an assertion error (see below). I got it
running a week or two ago, though.
I have zero knowledge on how to debug this, please ignore if
not helpful.
Ciao, Gregor
[...]
+ CFLAGS='-g -O2 -fdebug-prefix-map=/home/grfz/src/emacs=. -fstack-protector-strong -Wformat -Werror=format-security -Wall -fno-pie'
[...]
+ ./configure -C --prefix=/usr/local/stow/emacs-snapshot --with-file-notification=inotify --with-cairo --without-toolkit-scroll-bars --with-x-toolkit=gtk3 --with-sound=yes --without-gconf --with-mailutils --with-x=yes --enable-checking=yes --enable-check-lisp-object-type=yes --with-nativecomp
[...]
Dumping under the name emacs.pdmp
dumping fingerprint: 2cb0caf05c685124336e3e73f3c31aa2459767f514a8c849b145c03c0804d21b
Dump complete
Byte counts: header=96 hot=8085304 discardable=138024 cold=4042496
Reloc counts: hot=442357 discardable=4880
Adding name emacs-28.0.50.1
Adding name emacs-28.0.50.1.pdmp
cp -f emacs.pdmp bootstrap-emacs.pdmp
make[1]: Leaving directory '/home/grfz/src/emacs/src'
make -C lisp all
make[1]: Entering directory '/home/grfz/src/emacs/lisp'
make -C ../leim all EMACS="../src/emacs"
make -C ../admin/grammars all EMACS="../../src/emacs"
GEN cus-load.el
GEN finder-inf.el
make[2]: Entering directory '/home/grfz/src/emacs/admin/grammars'
make[2]: Nothing to be done for 'all'.
make[2]: Leaving directory '/home/grfz/src/emacs/admin/grammars'
make[2]: Entering directory '/home/grfz/src/emacs/leim'
make[2]: Nothing to be done for 'all'.
make[2]: Leaving directory '/home/grfz/src/emacs/leim'
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7fdfaa7f809b]
../src/emacs[0x43f5fa]
/bin/bash: line 2: 3545 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp -l cus-dep --eval '(setq generated-custom-dependencies-file (unmsys--file-name "./cus-load.el"))' -f custom-make-dependencies . ./calc ./calendar ./cedet ./cedet/ede ./cedet/semantic ./cedet/semantic/analyze ./cedet/semantic/bovine ./cedet/semantic/decorate ./cedet/semantic/symref ./cedet/semantic/wisent ./cedet/srecode ./eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./emacs-lisp ./emacs-lisp/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./emulation ./erc ./eshell ./gnus ./image ./international ./international/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./language ./language/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./leim ./leim/ja-dic ./leim/quail ./mail ./mh-e ./net ./nxml ./org ./play ./progmodes ./progmodes/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./term/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./textmodes ./textmodes/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./url ./vc ./vc/eln-x86_64-pc-linux-gnu-b5e14142f9b69778
make[1]: *** [Makefile:161: cus-load.el] Error 134
make[1]: *** Waiting for unfinished jobs....
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7fcabd83209b]
../src/emacs[0x43f5fa]
/bin/bash: line 2: 3550 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp -l finder --eval '(setq generated-finder-keywords-file (unmsys--file-name "./finder-inf.el"))' -f finder-compile-keywords-make-dist . ./calc ./calendar ./cedet ./cedet/ede ./cedet/semantic ./cedet/semantic/analyze ./cedet/semantic/bovine ./cedet/semantic/decorate ./cedet/semantic/symref ./cedet/semantic/wisent ./cedet/srecode ./eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./emacs-lisp ./emacs-lisp/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./emulation ./erc ./eshell ./gnus ./image ./international ./international/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./language ./language/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./mail ./mh-e ./net ./nxml ./org ./play ./progmodes ./progmodes/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./term/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./textmodes ./textmodes/eln-x86_64-pc-linux-gnu-b5e14142f9b69778 ./url ./vc ./vc/eln-x86_64-pc-linux-gnu-b5e14142f9b69778
make[1]: *** [Makefile:169: finder-inf.el] Error 134
make[2]: Entering directory '/home/grfz/src/emacs/lisp'
ELC+ELN emacs-lisp/eieio.elc
ELC+ELN emacs-lisp/eieio-base.elc
ELC+ELN cedet/semantic/db.elc
ELC+ELN align.elc
ELC+ELN allout.elc
ELC+ELN allout-widgets.elc
ELC+ELN ansi-color.elc
ELC+ELN apropos.elc
ELC+ELN arc-mode.elc
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7f547ddec09b]
../src/emacs[0x43f5fa]
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7f4ea24f009b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4987 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap allout.el
/bin/bash: line 1: 4980 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap emacs-lisp/eieio-base.el
make[2]: *** [Makefile:312: allout.elc] Error 134
make[2]: *** Waiting for unfinished jobs....
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7fb0c251609b]
../src/emacs[0x43f5fa]
make[2]: *** [Makefile:312: emacs-lisp/eieio-base.elc] Error 134
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7f398f23e09b]
../src/emacs[0x43f5fa]
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7feacb9b709b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4981 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap emacs-lisp/eieio.el
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
make[2]: *** [Makefile:312: emacs-lisp/eieio.elc] Error 134
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7f1a10d7e09b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4990 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap ansi-color.el
make[2]: *** [Makefile:312: ansi-color.elc] Error 134
/bin/bash: line 1: 4983 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap cedet/semantic/db.el
/bin/bash: line 1: 4985 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap align.el
make[2]: *** [Makefile:312: cedet/semantic/db.elc] Error 134
make[2]: *** [Makefile:312: align.elc] Error 134
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7ff53d3d709b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4993 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap arc-mode.el
make[2]: *** [Makefile:312: arc-mode.elc] Error 134
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7f80cba1f09b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4988 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap allout-widgets.el
make[2]: *** [Makefile:312: allout-widgets.elc] Error 134
comp.c:3725: Emacs fatal error: assertion failed: check_comp_unit_relocs (comp_u)
Backtrace:
../src/emacs[0x574f3b]
../src/emacs[0x42b836]
../src/emacs[0x430c0a]
../src/emacs[0x6374aa]
../src/emacs[0x5c9da6]
../src/emacs[0x5d0374]
../src/emacs[0x43e644]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb)[0x7fbe9772509b]
../src/emacs[0x43f5fa]
/bin/bash: line 1: 4992 Aborted EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap apropos.el
make[2]: *** [Makefile:312: apropos.elc] Error 134
make[2]: Leaving directory '/home/grfz/src/emacs/lisp'
make[1]: *** [Makefile:346: compile-main] Error 2
make[1]: Leaving directory '/home/grfz/src/emacs/lisp'
make: *** [Makefile:411: lisp] Error 2
+ exit
2
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 8
2020-05-16 18:22 ` Gregor Zattler
@ 2020-05-16 19:59 ` Andrea Corallo
2020-05-17 5:26 ` Gregor Zattler
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-05-16 19:59 UTC (permalink / raw)
To: emacs-devel
Gregor Zattler <telegraph@gmx.net> writes:
> Hi Andrea, emacs developers,
> * Andrea Corallo <akrl@sdf.org> [2020-05-16; 15:24]:
>> Please follow up with comments, suggestions on missing features or
>> observations
>
> I tried to build feature/native-comp as of today (commit
> d6f6353cfdbbea5501915675081265b4dc4591e3) on an up-to-date
> debian buster (10.4) with version 8.3.0-6 of libgccjit
> installed and got an assertion error (see below). I got it
> running a week or two ago, though.
>
> I have zero knowledge on how to debug this, please ignore if
> not helpful.
>
> Ciao, Gregor
Hi Gregor thanks for the feedback.
I changed quite a number of things so the issue is most likely to be real.
Could you please open a bug specifying your architecture and all the
issued commands for compiling? The best is to have a bug so we can
track it there.
Thanks in advanced
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 9
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
2020-05-16 16:21 ` Eli Zaretskii
2020-05-16 18:22 ` Gregor Zattler
@ 2020-06-14 13:23 ` Andrea Corallo
2020-06-14 17:23 ` arthur miller
` (3 more replies)
2 siblings, 4 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-06-14 13:23 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Stefan Monnier
Hi all,
I wrote an update about what is going on in the native-comp branch.
In summary is about the recent big improvements on compile time, lambda
performance and portability.
https://akrl.sdf.org/gccemacs.html
I believe in the current state the branch should be much more accessible
to 32bit systems and constrained hardware in general.
I think the next topics coming for me are dynamic scope compilation and
primitive function advising.
Regards
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* RE: On elisp running native - Update 9
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
@ 2020-06-14 17:23 ` arthur miller
2020-06-15 5:04 ` Gong-Yi Liao 廖宮毅
` (2 subsequent siblings)
3 siblings, 0 replies; 210+ messages in thread
From: arthur miller @ 2020-06-14 17:23 UTC (permalink / raw)
To: Andrea Corallo, emacs-devel@gnu.org; +Cc: Eli Zaretskii, Stefan Monnier
[-- Attachment #1: Type: text/plain, Size: 866 bytes --]
Whauh! That is a great work! Thanks.
Skickat från min Samsung Galaxy-smartphone.
-------- Originalmeddelande --------
Från: Andrea Corallo <akrl@sdf.org>
Datum: 2020-06-14 15:24 (GMT+01:00)
Till: emacs-devel@gnu.org
Kopia: Eli Zaretskii <eliz@gnu.org>, Stefan Monnier <monnier@iro.umontreal.ca>
Ämne: Re: On elisp running native - Update 9
Hi all,
I wrote an update about what is going on in the native-comp branch.
In summary is about the recent big improvements on compile time, lambda
performance and portability.
https://akrl.sdf.org/gccemacs.html
I believe in the current state the branch should be much more accessible
to 32bit systems and constrained hardware in general.
I think the next topics coming for me are dynamic scope compilation and
primitive function advising.
Regards
Andrea
--
akrl@sdf.org
[-- Attachment #2: Type: text/html, Size: 1849 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 9
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
2020-06-14 17:23 ` arthur miller
@ 2020-06-15 5:04 ` Gong-Yi Liao 廖宮毅
2020-06-15 7:41 ` Andrea Corallo
2020-06-17 23:54 ` Stefan Monnier
2020-07-16 20:00 ` On elisp running native - Update 10 Andrea Corallo via Emacs development discussions.
3 siblings, 1 reply; 210+ messages in thread
From: Gong-Yi Liao 廖宮毅 @ 2020-06-15 5:04 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
[-- Attachment #1: Type: text/plain, Size: 1254 bytes --]
Andrea,
Question:
What's the difference between Update 1's and Update 9's benchmark table's
byte code column?
The byte code column's run time results have decreased roughly 78% from
Update 1's 507.23s to Update 9's 114.91, that seems unbelievable if all the
Update 9's byte code result are purely based on byte code without any
natively compiled code.
Are Update 9's byte code result based on natively compiled built-in
function and byte code compiled benchmark functions or they are purely all
byte code compiled?
Or there are drastic changes in benchmark environment settings (hardware,
number of repetitions) betweens these two updates?
Thanks,
Gong-Yi.
On Sun, Jun 14, 2020, 8:24 AM Andrea Corallo <akrl@sdf.org> wrote:
> Hi all,
>
> I wrote an update about what is going on in the native-comp branch.
>
> In summary is about the recent big improvements on compile time, lambda
> performance and portability.
>
> https://akrl.sdf.org/gccemacs.html
>
> I believe in the current state the branch should be much more accessible
> to 32bit systems and constrained hardware in general.
>
> I think the next topics coming for me are dynamic scope compilation and
> primitive function advising.
>
> Regards
>
> Andrea
>
> --
> akrl@sdf.org
>
>
[-- Attachment #2: Type: text/html, Size: 2112 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 9
2020-06-15 5:04 ` Gong-Yi Liao 廖宮毅
@ 2020-06-15 7:41 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-06-15 7:41 UTC (permalink / raw)
To: Gong-Yi Liao 廖宮毅
Cc: Eli Zaretskii, Stefan Monnier, emacs-devel
Gong-Yi Liao 廖宮毅 <gongyi.liao@gmail.com> writes:
> Andrea,
>
> Question:
> What's the difference between Update 1's and Update 9's benchmark
> table's byte code column?
>
> The byte code column's run time results have decreased roughly 78%
> from Update 1's 507.23s to Update 9's 114.91, that seems unbelievable
> if all the Update 9's byte code result are purely based on byte code
> without any natively compiled code.
>
> Are Update 9's byte code result based on natively compiled built-in
> function and byte code compiled benchmark functions or they are
> purely all byte code compiled?
>
> Or there are drastic changes in benchmark environment settings
> (hardware, number of repetitions) betweens these two updates?
>
> Thanks,
> Gong-Yi.
Hi Gong-Yi,
The first measures were based on a collection of benchmarks that later
became what we have in ELPA now.
In the transition a number of things changed: the set slightly changed,
IIRC also the durations have been normalized to be more homogeneous and
the result is given in avg per iteration instead of being cumulated.
Lastly the two tests has been run on two different machines.
Probably would be interesting to re-add the u-benchmark for type-hints
in the future.
Regards
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 9
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
2020-06-14 17:23 ` arthur miller
2020-06-15 5:04 ` Gong-Yi Liao 廖宮毅
@ 2020-06-17 23:54 ` Stefan Monnier
2020-06-18 6:55 ` Andrea Corallo
2020-07-16 20:00 ` On elisp running native - Update 10 Andrea Corallo via Emacs development discussions.
3 siblings, 1 reply; 210+ messages in thread
From: Stefan Monnier @ 2020-06-17 23:54 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> I believe in the current state the branch should be much more accessible
> to 32bit systems and constrained hardware in general.
As a case in point, I just built it on 32bit i386 and armhf (on an
Orange Pi with 1GB of RAM) ;-)
For some reason, the armhf encountered a few crashes. Mostly some
segfault when compiling linux.el (but that segfault disappeared when
retrying after compiling everything else) and modula2.el and octave.el
both still fail with:
ELC+ELN progmodes/modula2.elc
Wrong type argument: listp, (:foundry unspecified :width unspecified :height unspecified :weight unspecified :slant unspecified ...)
make[2]: *** [Makefile:312: progmodes/modula2.elc] Error 255
ELC+ELN progmodes/octave.elc
Wrong type argument: listp, (/bin/bash : ligne 1 : 2978 Erreur de segmentation (core dumped)EMACSLOADPATH= '../src/emacs' -batch --no-site-file --no-site-lisp --eval '(setq load-prefer-newer t)' -l comp -f batch-byte-native-compile-for-bootstrap progmodes/octave.el
make[2]: *** [Makefile:312: progmodes/octave.elc] Error 139
-- Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 9
2020-06-17 23:54 ` Stefan Monnier
@ 2020-06-18 6:55 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-06-18 6:55 UTC (permalink / raw)
To: Stefan Monnier; +Cc: Eli Zaretskii, emacs-devel
Stefan Monnier <monnier@iro.umontreal.ca> writes:
>> I believe in the current state the branch should be much more accessible
>> to 32bit systems and constrained hardware in general.
>
> As a case in point, I just built it on 32bit i386 and armhf (on an
> Orange Pi with 1GB of RAM) ;-)
>
> For some reason, the armhf encountered a few crashes. Mostly some
> segfault when compiling linux.el (but that segfault disappeared when
> retrying after compiling everything else) and modula2.el and octave.el
> both still fail with:
Cool thanks for trying! Interesting never tried aarch32, I add it to my
todo list then.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 10
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
` (2 preceding siblings ...)
2020-06-17 23:54 ` Stefan Monnier
@ 2020-07-16 20:00 ` Andrea Corallo via Emacs development discussions.
2020-08-17 18:43 ` On elisp running native - Update 11 Andrea Corallo via Emacs development discussions.
3 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-07-16 20:00 UTC (permalink / raw)
To: emacs-devel; +Cc: Eli Zaretskii, Stefan Monnier, rms
Hi all,
I've added a new update about the native compiler branch at:
https://akrl.sdf.org/gccemacs.html
In case you are in a rush here is the very summary:
- Dynamic scope support is in.
- We now do a better job at optimizing pure functions.
I think next is trying to rework where and how we keep .eln files.
Ciao
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 11
2020-07-16 20:00 ` On elisp running native - Update 10 Andrea Corallo via Emacs development discussions.
@ 2020-08-17 18:43 ` Andrea Corallo via Emacs development discussions.
2020-08-18 5:13 ` Arthur Miller
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-08-17 18:43 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms
Hi all,
I've pushed the branch reorganizing the layout of the .eln files and the
load and compilation triggering mechanism.
Works for me, hope it does not introduce too much breakage.
I tried to summarize how it works now on my usual dev log page
http://akrl.sdf.org/gccemacs.html
Now the use should me more transparent to the user and legittimate us to
make it more transparent also API wise, removing some incompatibility
that may be still around on that side.
Also having now some more infrastructure on how and where to place .eln
files should be easier to tackle the advised primitives topic for which
I plan to generate some code.
Thanks!
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 11
2020-08-17 18:43 ` On elisp running native - Update 11 Andrea Corallo via Emacs development discussions.
@ 2020-08-18 5:13 ` Arthur Miller
2020-08-18 8:17 ` Andrea Corallo via Emacs development discussions.
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
1 sibling, 1 reply; 210+ messages in thread
From: Arthur Miller @ 2020-08-18 5:13 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms, Andrea Corallo
Andrea Corallo via "Emacs development discussions."
<emacs-devel@gnu.org> writes:
> Hi all,
>
> I've pushed the branch reorganizing the layout of the .eln files and the
> load and compilation triggering mechanism.
>
> Works for me, hope it does not introduce too much breakage.
>
> I tried to summarize how it works now on my usual dev log page
> http://akrl.sdf.org/gccemacs.html
Hi Andrea;
Thanks for great work, native compiler is really awesome and works for
the most part great, have being using it since you posted first update
and it works for the most part really well. I have noticed really big
difference since few weeks ago when I compiled both Emacs and elpa
folder to native.
I have some questions about this last update:
"Deferred compilation is now enabled by default, therefore is not
anymore necessary to set comp-deferred-compilation manually."
Can I still compile everything to native "up-front", without need to
wait for deffered compilation to kick in? It does not take so much time
for me to do it, so I prefer to do it all in advance.
"When a .elc file is being loaded if a suitable .eln file is found in
one of the eln-cache directories this is loaded instead. Otherwise the
file is compiled asyncronously and its definitions are swapped once
finished with that.
I've removed .eln from load-suffixes as loading native code is not
supposed to be manually done by the user. This has also the positive
side effect to reduce the number of files we search for while performing
a load."
> Also having now some more infrastructure on how and where to place .eln
> files should be easier to tackle the advised primitives topic for which
> I plan to generate some code.
Will there ever be possible for emacs to check if natively compiled file
is older then .el (source file) and do compilation (el->elc->eln)
automatically so we don't have to waste disk space and keep around .elc
files? Or did I missunderstand how is is supposed to work?
I would really prefer to see .eln as "equal" to .elc; and as there is
a 'some-file.el' and 'some-file.elc' to see 'some-file.eln' and .eln to
replace .elc as compiled .el file.
>
> Thanks!
Thank you for the great work!
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 11
2020-08-18 5:13 ` Arthur Miller
@ 2020-08-18 8:17 ` Andrea Corallo via Emacs development discussions.
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-08-18 8:17 UTC (permalink / raw)
To: Arthur Miller
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Arthur Miller <arthur.miller@live.com> writes:
> Andrea Corallo via "Emacs development discussions."
> <emacs-devel@gnu.org> writes:
>
>> Hi all,
>>
>> I've pushed the branch reorganizing the layout of the .eln files and the
>> load and compilation triggering mechanism.
>>
>> Works for me, hope it does not introduce too much breakage.
>>
>> I tried to summarize how it works now on my usual dev log page
>> http://akrl.sdf.org/gccemacs.html
> Hi Andrea;
>
> Thanks for great work, native compiler is really awesome and works for
> the most part great, have being using it since you posted first update
> and it works for the most part really well. I have noticed really big
> difference since few weeks ago when I compiled both Emacs and elpa
> folder to native.
Hi Arthur,
that's great to hear!
> I have some questions about this last update:
>
> "Deferred compilation is now enabled by default, therefore is not
> anymore necessary to set comp-deferred-compilation manually."
>
> Can I still compile everything to native "up-front", without need to
> wait for deffered compilation to kick in? It does not take so much time
> for me to do it, so I prefer to do it all in advance.
Yep. By default make still native compiles all Emacs (we may change the
default but I guess we'll leave a knob there). Regarding packages
something like ~(native-compile-async "~/.emacs.d/elpa/" 4 t)~ should
still do the job head of time.
> "When a .elc file is being loaded if a suitable .eln file is found in
> one of the eln-cache directories this is loaded instead. Otherwise the
> file is compiled asyncronously and its definitions are swapped once
> finished with that.
>
> I've removed .eln from load-suffixes as loading native code is not
> supposed to be manually done by the user. This has also the positive
> side effect to reduce the number of files we search for while performing
> a load."
>
>> Also having now some more infrastructure on how and where to place .eln
>> files should be easier to tackle the advised primitives topic for which
>> I plan to generate some code.
>
> Will there ever be possible for emacs to check if natively compiled file
> is older then .el (source file) and do compilation (el->elc->eln)
> automatically so we don't have to waste disk space and keep around .elc
> files? Or did I missunderstand how is is supposed to work?
>
> I would really prefer to see .eln as "equal" to .elc; and as there is
> a 'some-file.el' and 'some-file.elc' to see 'some-file.eln' and .eln to
> replace .elc as compiled .el file.
The short answer is: I think so.
I have an experimental branch where the .elc are not produced at all by
make bootstrap. The only complication is that for the Emacs build I had
to modify the process to depose files containing the doc so
make-docfile.c can eat those instead of the .elc files. Other than that
we should re-add .eln to load-suffixes. But as I'm not sure this is a
requirement I'd prefer first to converge with the current setup. Unless
I get some specific input on that I think I'll keep this idea and its
branch aside for now :)
Thanks
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-08-17 18:43 ` On elisp running native - Update 11 Andrea Corallo via Emacs development discussions.
2020-08-18 5:13 ` Arthur Miller
@ 2020-09-26 10:35 ` Andrea Corallo via Emacs development discussions.
2020-09-26 14:29 ` T.V Raman
` (4 more replies)
1 sibling, 5 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-26 10:35 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms
Hi all,
I've just updated my dev blog on the last feature/native-comp
developments <http://akrl.sdf.org/gccemacs.html>.
Outlines are:
- Fast bootstrap as default
- Exposing compiler type hints
- New libgccjit entry point
- New primitive trampoline mechanism
- Integration, turning the buoy
Please have a look if you are interested.
Regarding the last point I'm very much interested my-self in every
aspect where a native compiled Emacs behaves differently from vanilla.
The reason for that is that if this happens, well I've no a priori idea
of the possible reason as I'm done with my todo list! :)
Feedback, suggestions & bug reports are very welcome as usual.
Thanks!
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
@ 2020-09-26 14:29 ` T.V Raman
2020-09-26 14:56 ` Andrea Corallo via Emacs development discussions.
2020-09-27 10:35 ` Arthur Miller
` (3 subsequent siblings)
4 siblings, 1 reply; 210+ messages in thread
From: T.V Raman @ 2020-09-26 14:29 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Andrea Corallo, Eli Zaretskii, Stefan Monnier, rms
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=gb18030, Size: 621 bytes --]
Hi Andrea,
This looks awesome.
A couple of questions:
Now that you handle the optimization declarations from cl, could you
also handle cl-declaim of the same --- typically used in CL at
top-level / start-of file?
(cl-declaim (optimize (safety 0) (speed 3)))
2. In your example for advising primitives, you show the generate
dfunction as
(defun --subr-trampoline-delete-file (filename &optional trash)
(funcall filename trash))
Should the above check if delete-file was called interactively, and
if yes, in turn call funcall-interactively?
--
7©4 Id: kg:/m/0285kf1 •0Ü8
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-26 14:29 ` T.V Raman
@ 2020-09-26 14:56 ` Andrea Corallo via Emacs development discussions.
2020-09-26 15:07 ` T.V Raman
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-26 14:56 UTC (permalink / raw)
To: T.V Raman
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
"T.V Raman" <raman@google.com> writes:
> Hi Andrea,
Hi!
> This looks awesome.
>
> A couple of questions:
>
> Now that you handle the optimization declarations from cl, could you
> also handle cl-declaim of the same --- typically used in CL at
> top-level / start-of file?
>
>
> (cl-declaim (optimize (safety 0) (speed 3)))
ATM `cl-declaim' (as `cl-proclaim') are not effective regarding the
speed parameter of the native compiler. To a quick look to the
HyperSpec I guess `cl-declaim' should just set `comp-speed' other than
what is already doing, shall do it.
> 2. In your example for advising primitives, you show the generate
> dfunction as
> (defun --subr-trampoline-delete-file (filename &optional trash)
> (funcall filename trash))
>
> Should the above check if delete-file was called interactively, and
> if yes, in turn call funcall-interactively?
Interesting, I think should be fine like it is now as Lisp code is
calling explicitly `funcall-interactively' anyway.
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-26 14:56 ` Andrea Corallo via Emacs development discussions.
@ 2020-09-26 15:07 ` T.V Raman
2020-09-28 12:22 ` Lars Brinkhoff
0 siblings, 1 reply; 210+ messages in thread
From: T.V Raman @ 2020-09-26 15:07 UTC (permalink / raw)
To: Andrea Corallo
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=gb18030, Size: 1580 bytes --]
Andrea Corallo <akrl@sdf.org> writes:
SG!
to be honest I never could remember the difference between cl-declaim
and cl-proclaim when I used Common Lisp heavily.
You said you'd handle the compilation speed, assume that automatically
sets the highest level of optimization?
I built the native-emacs branch earlier this morning and did a few
timing tests by having EWW render some large epub files, I didn't notice
much difference in timing.
> "T.V Raman" <raman@google.com> writes:
>
>> Hi Andrea,
>
> Hi!
>
>> This looks awesome.
>>
>> A couple of questions:
>>
>> Now that you handle the optimization declarations from cl, could you
>> also handle cl-declaim of the same --- typically used in CL at
>> top-level / start-of file?
>>
>>
>> (cl-declaim (optimize (safety 0) (speed 3)))
>
> ATM `cl-declaim' (as `cl-proclaim') are not effective regarding the
> speed parameter of the native compiler. To a quick look to the
> HyperSpec I guess `cl-declaim' should just set `comp-speed' other than
> what is already doing, shall do it.
>
>> 2. In your example for advising primitives, you show the generate
>> dfunction as
>> (defun --subr-trampoline-delete-file (filename &optional trash)
>> (funcall filename trash))
>>
>> Should the above check if delete-file was called interactively, and
>> if yes, in turn call funcall-interactively?
>
> Interesting, I think should be fine like it is now as Lisp code is
> calling explicitly `funcall-interactively' anyway.
>
> Andrea
--
7©4 Id: kg:/m/0285kf1 •0Ü8
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-26 15:07 ` T.V Raman
@ 2020-09-28 12:22 ` Lars Brinkhoff
2020-09-28 14:03 ` T.V Raman
0 siblings, 1 reply; 210+ messages in thread
From: Lars Brinkhoff @ 2020-09-28 12:22 UTC (permalink / raw)
To: emacs-devel
T.V Raman wrote:
> Andrea Corallo wrote:
>> ATM `cl-declaim' (as `cl-proclaim') are not effective regarding the
>> speed parameter of the native compiler. To a quick look to the
>> HyperSpec I guess `cl-declaim' should just set `comp-speed' other
>> than what is already doing, shall do it.
> to be honest I never could remember the difference between cl-declaim
> and cl-proclaim when I used Common Lisp heavily.
In Common Lisp PROCLAIM is just a function so will only have an effect
when it's called, typically at load time or run time. When DECLAIM
appears at top level, the compiler should take notice and update its
state at compile time.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
2020-09-26 14:29 ` T.V Raman
@ 2020-09-27 10:35 ` Arthur Miller
2020-09-27 18:28 ` Andrea Corallo via Emacs development discussions.
2020-09-28 14:10 ` Yuan Fu
` (2 subsequent siblings)
4 siblings, 1 reply; 210+ messages in thread
From: Arthur Miller @ 2020-09-27 10:35 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms, Andrea Corallo
Andrea Corallo via "Emacs development discussions."
<emacs-devel@gnu.org> writes:
> Hi all,
>
> I've just updated my dev blog on the last feature/native-comp
> developments <http://akrl.sdf.org/gccemacs.html>.
>
> Outlines are:
> - Fast bootstrap as default
> - Exposing compiler type hints
> - New libgccjit entry point
> - New primitive trampoline mechanism
> - Integration, turning the buoy
>
> Please have a look if you are interested.
>
> Regarding the last point I'm very much interested my-self in every
> aspect where a native compiled Emacs behaves differently from vanilla.
> The reason for that is that if this happens, well I've no a priori idea
> of the possible reason as I'm done with my todo list! :)
>
> Feedback, suggestions & bug reports are very welcome as usual.
>
> Thanks!
>
> Andrea
Hi Andrea! Thanks for the great work and update, looks awesome as
always!
I have just compiled Emacs, and am runing it, it seems to work fine. I
have though at question:
Are we going to see this in master in Emacs 28.0? :-)
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-27 10:35 ` Arthur Miller
@ 2020-09-27 18:28 ` Andrea Corallo via Emacs development discussions.
2020-09-27 18:30 ` Lars Ingebrigtsen
2020-09-27 20:37 ` T.V Raman
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-27 18:28 UTC (permalink / raw)
To: Arthur Miller
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Arthur Miller <arthur.miller@live.com> writes:
> Hi Andrea! Thanks for the great work and update, looks awesome as
> always!
Hi Arthur, thanks for the kind words :)
> I have just compiled Emacs, and am runing it, it seems to work fine. I
> have though at question:
>
> Are we going to see this in master in Emacs 28.0? :-)
Well I guess this does not depend (at least entirely) on me. I'd really
like to see it in 28 as I think is useful to a number of users.
I'll just do my best and seek for the necessary guidance on this.
Thanks!
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-27 18:28 ` Andrea Corallo via Emacs development discussions.
@ 2020-09-27 18:30 ` Lars Ingebrigtsen
2020-09-27 20:39 ` T.V Raman
` (2 more replies)
2020-09-27 20:37 ` T.V Raman
1 sibling, 3 replies; 210+ messages in thread
From: Lars Ingebrigtsen @ 2020-09-27 18:30 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, rms, Stefan Monnier, Arthur Miller, Andrea Corallo
Andrea Corallo via "Emacs development discussions."
<emacs-devel@gnu.org> writes:
> Well I guess this does not depend (at least entirely) on me. I'd really
> like to see it in 28 as I think is useful to a number of users.
I'd definitely like to see it in Emacs 28.
--
(domestic pets only, the antidote for overdose, milk.)
bloggy blog: http://lars.ingebrigtsen.no
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-27 18:30 ` Lars Ingebrigtsen
@ 2020-09-27 20:39 ` T.V Raman
2020-09-27 22:10 ` Stefan Monnier
2020-09-28 6:06 ` Eli Zaretskii
2 siblings, 0 replies; 210+ messages in thread
From: T.V Raman @ 2020-09-27 20:39 UTC (permalink / raw)
To: Lars Ingebrigtsen
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
rms, Stefan Monnier, Arthur Miller, Andrea Corallo
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=gb18030, Size: 659 bytes --]
Lars Ingebrigtsen <larsi@gnus.org> writes:
That is encouraging.
Would be nice if native compilation could speed up EWW on large HTML
files --- I use EWW to read EBooks.
A good test case for measuring speed is the HTML5 specification, it's a
giant document (approx 20,000 lines in the rendered buffer)
> Andrea Corallo via "Emacs development discussions."
> <emacs-devel@gnu.org> writes:
>
>> Well I guess this does not depend (at least entirely) on me. I'd really
>> like to see it in 28 as I think is useful to a number of users.
>
> I'd definitely like to see it in Emacs 28.
--
Thanks,
--Raman
7©4 Id: kg:/m/0285kf1 •0Ü8
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-27 18:30 ` Lars Ingebrigtsen
2020-09-27 20:39 ` T.V Raman
@ 2020-09-27 22:10 ` Stefan Monnier
2020-09-28 6:06 ` Eli Zaretskii
2 siblings, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-09-27 22:10 UTC (permalink / raw)
To: Lars Ingebrigtsen
Cc: Eli Zaretskii, Andrea Corallo, rms, Arthur Miller,
Andrea Corallo via Emacs development discussions.
>> Well I guess this does not depend (at least entirely) on me. I'd really
>> like to see it in 28 as I think is useful to a number of users.
> I'd definitely like to see it in Emacs 28.
I have the impression that the code is pretty much ready to be merged
into `master` as a new experimental feature (obviously, it'll require
some tweaks since the current native-comp branch enables that feature
by default whereas we'd want to keep it disabled by default at first on
`master`).
IOW someone needs to take a close look at the diff between trunk and the
branch and make sure that its changes are nicely modularised, and once
that's done, then write a good commit message for it ;-)
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-27 18:30 ` Lars Ingebrigtsen
2020-09-27 20:39 ` T.V Raman
2020-09-27 22:10 ` Stefan Monnier
@ 2020-09-28 6:06 ` Eli Zaretskii
2020-09-28 10:28 ` Andrea Corallo via Emacs development discussions.
2 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-09-28 6:06 UTC (permalink / raw)
To: Lars Ingebrigtsen; +Cc: arthur.miller, akrl, rms, monnier, emacs-devel
> From: Lars Ingebrigtsen <larsi@gnus.org>
> Date: Sun, 27 Sep 2020 20:30:44 +0200
> Cc: Eli Zaretskii <eliz@gnu.org>, rms@gnu.org,
> Stefan Monnier <monnier@iro.umontreal.ca>,
> Arthur Miller <arthur.miller@live.com>, Andrea Corallo <akrl@sdf.org>
>
> Andrea Corallo via "Emacs development discussions."
> <emacs-devel@gnu.org> writes:
>
> > Well I guess this does not depend (at least entirely) on me. I'd really
> > like to see it in 28 as I think is useful to a number of users.
>
> I'd definitely like to see it in Emacs 28.
Given that Emacs 28 is still quite far away, I see no reason why it
couldn't include this important feature.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 6:06 ` Eli Zaretskii
@ 2020-09-28 10:28 ` Andrea Corallo via Emacs development discussions.
2020-09-28 10:45 ` Eli Zaretskii
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 10:28 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: Lars Ingebrigtsen, emacs-devel, rms, monnier, arthur.miller
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Lars Ingebrigtsen <larsi@gnus.org>
>> Date: Sun, 27 Sep 2020 20:30:44 +0200
>> Cc: Eli Zaretskii <eliz@gnu.org>, rms@gnu.org,
>> Stefan Monnier <monnier@iro.umontreal.ca>,
>> Arthur Miller <arthur.miller@live.com>, Andrea Corallo <akrl@sdf.org>
>>
>> Andrea Corallo via "Emacs development discussions."
>> <emacs-devel@gnu.org> writes:
>>
>> > Well I guess this does not depend (at least entirely) on me. I'd really
>> > like to see it in 28 as I think is useful to a number of users.
>>
>> I'd definitely like to see it in Emacs 28.
>
> Given that Emacs 28 is still quite far away, I see no reason why it
> couldn't include this important feature.
Hi Eli Lars Stefan,
thanks for your feedback.
I suggest I re-have a look to the diff against master my-self and then
when done open a bug feature-request so we can handle the process and
review there. Sounds like the correct way?
Thanks
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 10:28 ` Andrea Corallo via Emacs development discussions.
@ 2020-09-28 10:45 ` Eli Zaretskii
2020-09-28 11:02 ` Andrea Corallo via Emacs development discussions.
0 siblings, 1 reply; 210+ messages in thread
From: Eli Zaretskii @ 2020-09-28 10:45 UTC (permalink / raw)
To: Andrea Corallo; +Cc: arthur.miller, larsi, rms, monnier, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Cc: Lars Ingebrigtsen <larsi@gnus.org>, emacs-devel@gnu.org, rms@gnu.org,
> monnier@iro.umontreal.ca, arthur.miller@live.com
> Date: Mon, 28 Sep 2020 10:28:58 +0000
>
> I suggest I re-have a look to the diff against master my-self and then
> when done open a bug feature-request so we can handle the process and
> review there. Sounds like the correct way?
Yes, please.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 10:45 ` Eli Zaretskii
@ 2020-09-28 11:02 ` Andrea Corallo via Emacs development discussions.
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 11:02 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: larsi, emacs-devel, rms, monnier, arthur.miller
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Cc: Lars Ingebrigtsen <larsi@gnus.org>, emacs-devel@gnu.org, rms@gnu.org,
>> monnier@iro.umontreal.ca, arthur.miller@live.com
>> Date: Mon, 28 Sep 2020 10:28:58 +0000
>>
>> I suggest I re-have a look to the diff against master my-self and then
>> when done open a bug feature-request so we can handle the process and
>> review there. Sounds like the correct way?
>
> Yes, please.
Great, will do.
Thanks
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-27 18:28 ` Andrea Corallo via Emacs development discussions.
2020-09-27 18:30 ` Lars Ingebrigtsen
@ 2020-09-27 20:37 ` T.V Raman
1 sibling, 0 replies; 210+ messages in thread
From: T.V Raman @ 2020-09-27 20:37 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Arthur Miller, Andrea Corallo, Eli Zaretskii, Stefan Monnier, rms
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=gb18030, Size: 324 bytes --]
Hi Andrea, Happy to report that as one of those packages that advice
heavily, emacspeak is working well under the new gccemacs build.
I still get the warnings I alluded to a few days ago on startup re
compilation but things appear to function well after.
--
Thanks,
--Raman
7©4 Id: kg:/m/0285kf1 •0Ü8
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
2020-09-26 14:29 ` T.V Raman
2020-09-27 10:35 ` Arthur Miller
@ 2020-09-28 14:10 ` Yuan Fu
2020-09-28 14:51 ` Andrea Corallo via Emacs development discussions.
2020-09-28 15:19 ` Caio Henrique
2021-01-01 18:37 ` On elisp running native - Update 13 Andrea Corallo via Emacs development discussions.
4 siblings, 1 reply; 210+ messages in thread
From: Yuan Fu @ 2020-09-28 14:10 UTC (permalink / raw)
To: Andrea Corallo
Cc: Eli Zaretskii, Stefan Monnier, rms,
Andrea Corallo via Emacs development discussions.
Hi Andrea,
Update 12 looks very cool. Is it possible now to eliminate funcall to all built-in functions (instead of only primitives)? Do you think that it will give significant speed-up?
Yuan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 14:10 ` Yuan Fu
@ 2020-09-28 14:51 ` Andrea Corallo via Emacs development discussions.
2020-09-28 15:20 ` Yuan Fu
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 14:51 UTC (permalink / raw)
To: Yuan Fu
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Yuan Fu <casouri@gmail.com> writes:
> Hi Andrea,
>
> Update 12 looks very cool. Is it possible now to eliminate funcall to
> all built-in functions (instead of only primitives)? Do you think that
> it will give significant speed-up?
Hi Yuan,
in my nomenclature primitive functions are already all C functions. I
always thought that's way to call them [1]. But coming to your question
we already remove funcall calling all C functions so... no more
improvements there ;)
Andrea
[1] <https://www.gnu.org/software/emacs/manual/html_node/eintr/Primitive-Functions.html>
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 14:51 ` Andrea Corallo via Emacs development discussions.
@ 2020-09-28 15:20 ` Yuan Fu
2020-09-28 15:32 ` Andrea Corallo via Emacs development discussions.
0 siblings, 1 reply; 210+ messages in thread
From: Yuan Fu @ 2020-09-28 15:20 UTC (permalink / raw)
To: Andrea Corallo
Cc: Eli Zaretskii, Stefan Monnier, rms,
Andrea Corallo via Emacs development discussions.
> On Sep 28, 2020, at 10:51 AM, Andrea Corallo <akrl@sdf.org> wrote:
>
> Yuan Fu <casouri@gmail.com> writes:
>
>> Hi Andrea,
>>
>> Update 12 looks very cool. Is it possible now to eliminate funcall to
>> all built-in functions (instead of only primitives)? Do you think that
>> it will give significant speed-up?
>
> Hi Yuan,
>
> in my nomenclature primitive functions are already all C functions. I
> always thought that's way to call them [1]. But coming to your question
> we already remove funcall calling all C functions so... no more
> improvements there ;)
>
> Andrea
>
> [1] <https://www.gnu.org/software/emacs/manual/html_node/eintr/Primitive-Functions.html>
Thanks for clarifying. I got the false impression that normal compiled Elisp functions can also be called without funcall when watching your talk, or I just remembered wrong. :-)
Yuan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 15:20 ` Yuan Fu
@ 2020-09-28 15:32 ` Andrea Corallo via Emacs development discussions.
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 15:32 UTC (permalink / raw)
To: Yuan Fu
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Yuan Fu <casouri@gmail.com> writes:
>> On Sep 28, 2020, at 10:51 AM, Andrea Corallo <akrl@sdf.org> wrote:
>>
>> Yuan Fu <casouri@gmail.com> writes:
>>
>>> Hi Andrea,
>>>
>>> Update 12 looks very cool. Is it possible now to eliminate funcall to
>>> all built-in functions (instead of only primitives)? Do you think that
>>> it will give significant speed-up?
>>
>> Hi Yuan,
>>
>> in my nomenclature primitive functions are already all C functions. I
>> always thought that's way to call them [1]. But coming to your question
>> we already remove funcall calling all C functions so... no more
>> improvements there ;)
>>
>> Andrea
>>
>> [1] <https://www.gnu.org/software/emacs/manual/html_node/eintr/Primitive-Functions.html>
>
> Thanks for clarifying. I got the false impression that normal compiled
> Elisp functions can also be called without funcall when watching your
> talk, or I just remembered wrong. :-)
You remeber correctly, some C functions can be called directly from Lisp
if byte-compiled, but only the ones that got an op-code assigned in
bytecode. If you are curious see
<https://rocky.github.io/elisp-bytecode.pdf>. Native code compilation
allow for doing direct calls always as the problem of partitioning the
op byte-code space is not anymore there.
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
` (2 preceding siblings ...)
2020-09-28 14:10 ` Yuan Fu
@ 2020-09-28 15:19 ` Caio Henrique
2020-09-28 15:39 ` Andrea Corallo via Emacs development discussions.
2021-01-01 18:37 ` On elisp running native - Update 13 Andrea Corallo via Emacs development discussions.
4 siblings, 1 reply; 210+ messages in thread
From: Caio Henrique @ 2020-09-28 15:19 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms, Andrea Corallo
Hi Andrea,
I have a question: does libgccjit include support for the GCC's OpenMP
implementation? If so, then could that be used on Emacs? I.e. the elisp
equivalent of something like:
#pragma omp parallel for private(i) shared(a)
for (i=1; i<n; i++)
b[i] = (a[i] + a[i-1]);
Thanks,
Caio Henrique
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native - Update 12
2020-09-28 15:19 ` Caio Henrique
@ 2020-09-28 15:39 ` Andrea Corallo via Emacs development discussions.
2020-09-28 16:12 ` Caio Henrique
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 15:39 UTC (permalink / raw)
To: Caio Henrique
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Caio Henrique <caiohcs0@gmail.com> writes:
> Hi Andrea,
>
> I have a question: does libgccjit include support for the GCC's OpenMP
> implementation? If so, then could that be used on Emacs? I.e. the elisp
> equivalent of something like:
>
> #pragma omp parallel for private(i) shared(a)
> for (i=1; i<n; i++)
> b[i] = (a[i] + a[i-1]);
Hi Caio,
It does not. We may add it tho, but I think would be more beneficial to
expose first aref aset and see what we can generate with the GCC
vectorizer. But I guess before that in the line unboxing is waiting.
There is really a vast space of possible improvements to be covered for
the future.
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 15:39 ` Andrea Corallo via Emacs development discussions.
@ 2020-09-28 16:12 ` Caio Henrique
2020-09-28 17:21 ` Andrea Corallo via Emacs development discussions.
0 siblings, 1 reply; 210+ messages in thread
From: Caio Henrique @ 2020-09-28 16:12 UTC (permalink / raw)
To: Andrea Corallo
Cc: Eli Zaretskii, Caio Henrique, Stefan Monnier, rms,
Andrea Corallo via Emacs development discussions.
Andrea Corallo <akrl@sdf.org> writes:
> We may add it tho, but I think would be more beneficial to
> expose first aref aset and see what we can generate with the GCC
> vectorizer.
Does that mean that native elisp would be able to use SSE, AVX etc?
> There is really a vast space of possible improvements to be covered for
> the future.
Thanks for your hard work. :)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 12
2020-09-28 16:12 ` Caio Henrique
@ 2020-09-28 17:21 ` Andrea Corallo via Emacs development discussions.
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2020-09-28 17:21 UTC (permalink / raw)
To: Caio Henrique
Cc: Andrea Corallo via Emacs development discussions., Eli Zaretskii,
Stefan Monnier, rms
Caio Henrique <caiohcs0@gmail.com> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> We may add it tho, but I think would be more beneficial to
>> expose first aref aset and see what we can generate with the GCC
>> vectorizer.
>
> Does that mean that native elisp would be able to use SSE, AVX etc?
I must say I'm very ignorant regarding the X86 GCC backend so I'm not
sure which instructions (if any) can be generated with autovectorization
there. I've got the idea as these days I had to put my hands in the
vectorizer trying to improve it looking at Aarch64 SVE as output.
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native - Update 13
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
` (3 preceding siblings ...)
2020-09-28 15:19 ` Caio Henrique
@ 2021-01-01 18:37 ` Andrea Corallo via Emacs development discussions.
2021-01-01 20:07 ` tomas
4 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo via Emacs development discussions. @ 2021-01-01 18:37 UTC (permalink / raw)
To: Andrea Corallo via Emacs development discussions.
Cc: Eli Zaretskii, Stefan Monnier, rms
Hi all,
I wrote a new update on my dev blog regarding feature/native-comp
activity. This is about the recent value/type inference improvements.
Have a look if you are interested as I think it may open few doors.
<http://akrl.sdf.org/gccemacs.html>
Please feel free to feedback any question or comment.
Thanks!
Andrea
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 12:41 ` Andrea Corallo
2020-01-01 15:14 ` Stefan Monnier
@ 2020-01-01 15:19 ` Óscar Fuentes
2020-01-01 18:48 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Óscar Fuentes @ 2020-01-01 15:19 UTC (permalink / raw)
To: emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> To test it configure it with --with-nativecomp (this is disabled by
> default). I did some work and the normal build should be compilable and
> functional again.
I know you are doing what was requested, but it is pointless to set the
default of --with-nativecomp to false when the whole point of that
experimental branch is to develop and test that feature.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-01-01 15:19 ` On elisp running native Óscar Fuentes
@ 2020-01-01 18:48 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-01-01 18:48 UTC (permalink / raw)
To: Óscar Fuentes; +Cc: emacs-devel
Óscar Fuentes <ofv@wanadoo.es> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> To test it configure it with --with-nativecomp (this is disabled by
>> default). I did some work and the normal build should be compilable and
>> functional again.
>
> I know you are doing what was requested, but it is pointless to set the
> default of --with-nativecomp to false when the whole point of that
> experimental branch is to develop and test that feature.
>
Had the same thought... but I can cope with that.
On the other hand is quite easy to notice the problem. If the
compilation does not feel C++ish you know something is wrong :)
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-12-31 10:24 ` Andrea Corallo
2019-12-31 15:51 ` Eli Zaretskii
@ 2020-01-01 14:59 ` Stefan Monnier
1 sibling, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-01-01 14:59 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Eli Zaretskii, emacs-devel
> Should I squash all my history or keep it as it is? The branch went
> through quite some experimentations. I'm not sure about its history
> value.
The decision to squash or not is important (for us maintainers) when you
commit to `master`, but until then it's quite secondary.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2019-11-28 6:54 On elisp running native Andrea Corallo
` (3 preceding siblings ...)
2019-12-08 20:18 ` Andrea Corallo
@ 2020-03-15 14:21 ` Spenser Truex
2020-03-15 20:10 ` Andrea Corallo
4 siblings, 1 reply; 210+ messages in thread
From: Spenser Truex @ 2020-03-15 14:21 UTC (permalink / raw)
To: Andrea Corallo, emacs-devel
Andrea Corallo <akrl@sdf.org> writes:
> In case this is of interest I'd love to see it as feature branch to
> better collect feedback and work on its evolution.
I have a comment about `1.6 Compiler hints' http://akrl.sdf.org/gccemacs.html#org7d51b3b
I think it is better to avoid using separate symbol for each hint, and instead
use on that looks like a normal prefix notation.
(comp-hint-cons x) --> (comp-hint cons x)
This keeps the namespace cleaner. Also, consider the situation of specifying
multiple types, which may be desired at some point:
(comp-hint (cons fixnum...) x)
which can't be done symbolically
(comp-hint-cons-fixnum-...? x)
(comp-hint-fixnum-cons...? x)
All this compiler stuff is way over my head, so if my syntactic taste is
overshadowed by a more important consideration I will not be surprised.
Hope it helps.
> Andrea
--
Spenser Truex
spensertruex.com
San Francisco, USA
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-15 14:21 ` Spenser Truex
@ 2020-03-15 20:10 ` Andrea Corallo
2020-03-18 14:58 ` Lars Brinkhoff
0 siblings, 1 reply; 210+ messages in thread
From: Andrea Corallo @ 2020-03-15 20:10 UTC (permalink / raw)
To: Spenser Truex; +Cc: emacs-devel
Spenser Truex <lists@spensertruex.com> writes:
> Andrea Corallo <akrl@sdf.org> writes:
>
>> In case this is of interest I'd love to see it as feature branch to
>> better collect feedback and work on its evolution.
>
> I have a comment about `1.6 Compiler hints' http://akrl.sdf.org/gccemacs.html#org7d51b3b
>
> I think it is better to avoid using separate symbol for each hint, and instead
> use on that looks like a normal prefix notation.
>
> (comp-hint-cons x) --> (comp-hint cons x)
>
> This keeps the namespace cleaner. Also, consider the situation of specifying
> multiple types, which may be desired at some point:
>
> (comp-hint (cons fixnum...) x)
>
> which can't be done symbolically
>
> (comp-hint-cons-fixnum-...? x)
> (comp-hint-fixnum-cons...? x)
>
> All this compiler stuff is way over my head, so if my syntactic taste is
> overshadowed by a more important consideration I will not be surprised.
Hi Spenser,
two thoughts on this:
- `comp-hint-cons' and `comp-hint-fixnum' are imagined to be produced by
macro expansion and not meant to be used directly by the user.
- As of today the compiler is not that smart to take advantage of
something that can potentially more then one type and I don't know if
it will ever be, at least for the code generation side. Perhaps could
be useful for code analysis and diagnostic.
That said you are right this is not really flexible or general and I
think your suggestion is correct.
Thanks!
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* RE: On elisp running native
@ 2020-01-02 21:47 arthur miller
0 siblings, 0 replies; 210+ messages in thread
From: arthur miller @ 2020-01-02 21:47 UTC (permalink / raw)
To: Ergus, Andrea Corallo; +Cc: Eli Zaretskii, Stefan Monnier, emacs-devel@gnu.org
[-- Attachment #1: Type: text/plain, Size: 603 bytes --]
Skickat från min Samsung Galaxy-smartphone.
> Maybe a bit orthogonal to where the thread is
> going. But I have seen
> that libgccjit is in alpha stage and has been like
> that for years
> (according to the page). The source is included
> in gcc, but not compiled
> by default in any distribution. I don't find
> distributions providing it
> with the gcc bundle (I can ask for it's inclusion in > arch, but I have no
> idea about the others).
Just as info if you are Arch user: libgccjit is in AUR. Compiled without problems when I tried it, but it some time to compile.
[-- Attachment #2: Type: text/html, Size: 1433 bytes --]
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
@ 2020-03-06 13:28 Zhu Zihao
2020-03-06 17:51 ` Andrea Corallo
0 siblings, 1 reply; 210+ messages in thread
From: Zhu Zihao @ 2020-03-06 13:28 UTC (permalink / raw)
To: akrl; +Cc: emacs-devel
Hi Andrea! I noticed that you said type annotation is important for speed. Can
we do static analysing after macro-expansion to infer types? Maybe we can just
write types for function arg and return value, and infer types automatically for
local variables.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-06 13:28 Zhu Zihao
@ 2020-03-06 17:51 ` Andrea Corallo
2020-03-09 17:55 ` Zhu Zihao
2020-03-10 16:15 ` Zhu Zihao
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-06 17:51 UTC (permalink / raw)
To: Zhu Zihao; +Cc: emacs-devel
Zhu Zihao <all_but_last@163.com> writes:
> Hi Andrea! I noticed that you said type annotation is important for speed. Can
> we do static analysing after macro-expansion to infer types?
Hi Zhu,
yes this is how it works already both for types and values.
> Maybe we can just write types for function arg...
I think just some macro would be required to expand in something like:
(defun foo (x y)
(let ((x (comp-hint-fixnum y))
(y (comp-hint-fixnum y)))
...))
I suspect somebody could already have some ideas on the subject... :)
That said bare in mind that this is most likely non sufficent to exploit
all possible optimization because you likely want to hint the compiler
that the result of some additions in the body is still a fixnum and does
not overflow in the integers (this is just and example).
For the return value and type currently we don't do interprocedural
analysis but is a good suggestion and should not be hard to add. That
said I think would be fair to apply it only at speed 3.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread* Re: On elisp running native
2020-03-06 17:51 ` Andrea Corallo
@ 2020-03-09 17:55 ` Zhu Zihao
2020-03-09 18:52 ` Andrea Corallo
2020-03-10 16:15 ` Zhu Zihao
1 sibling, 1 reply; 210+ messages in thread
From: Zhu Zihao @ 2020-03-09 17:55 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Zhu Zihao, emacs-devel
I try to build native-comp branch on Archlinux. It works as expected in the
early stage of bootstrap. But when bootstrapping leim/ja-dic/ja-dic.el with
output:
ELC+ELN leim/ja-dic/ja-dic.elc
...
...
INFO Extracting OKURI-NASI entries...done
Emacs seems to be frozen and consuming my memories little by little. And
almost 12G RAM was consumed by Emacs when I totally terminate the process.
My GCC version is https://gcc.gnu.org/git/gitweb.cgi?p=gcc.git;a=commit;h=6957d3e4eef1f4243eb23ff62aea06139ef4415a
both libgccjit and gcc was compiled using this commit.
Have you exprienced this condition? Or is there any way to ignore some file in
native-compiling? (like no-byte-compile: t file local variable)
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 17:55 ` Zhu Zihao
@ 2020-03-09 18:52 ` Andrea Corallo
2020-03-09 19:14 ` Zhu Zihao
2020-03-09 19:18 ` Eli Zaretskii
0 siblings, 2 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-09 18:52 UTC (permalink / raw)
To: Zhu Zihao; +Cc: emacs-devel
Hi Zhu Zihao,
Zhu Zihao <all_but_last@163.com> writes:
> I try to build native-comp branch on Archlinux. It works as expected in the
> early stage of bootstrap. But when bootstrapping leim/ja-dic/ja-dic.el with
> output:
>
> ELC+ELN leim/ja-dic/ja-dic.elc
> ...
> ...
> INFO Extracting OKURI-NASI entries...done
>
> Emacs seems to be frozen and consuming my memories little by little. And
> almost 12G RAM was consumed by Emacs when I totally terminate the process.
I believe you had just to wait and all was working. leim file are the
hardest for GCC to optimize but the most was already compiled. You can
just restart from there.
> My GCC version is https://gcc.gnu.org/git/gitweb.cgi?p=gcc.git;a=commit;h=6957d3e4eef1f4243eb23ff62aea06139ef4415a
> both libgccjit and gcc was compiled using this commit.
>
> Have you exprienced this condition?
I've never measured the max memory consumption of the while bootstrap
but I usually count like 2GB for each compilation unit when optimizing.
Certanly some can be more demanding.
Anyway indeed the total memory consumption depends on the parallelism
you choose for building. On my main dev machine I've 32GB of ram and I
routinary build speed 2 at -j16 without problems.
To have a very raw estimation of how long it takes speed 2 you can
divide 10h by the number of CPUs you dedicate to the compilation.
> Or is there any way to ignore some file in
> native-compiling? (like no-byte-compile: t file local variable)
Not for now.
Should improve a lot when I get to implement the deferred compilation.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 18:52 ` Andrea Corallo
@ 2020-03-09 19:14 ` Zhu Zihao
2020-03-09 19:41 ` Andrea Corallo
2020-03-09 21:06 ` Stefan Monnier
2020-03-09 19:18 ` Eli Zaretskii
1 sibling, 2 replies; 210+ messages in thread
From: Zhu Zihao @ 2020-03-09 19:14 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Zhu Zihao, Stefan Monnier, emacs-devel
Thanks for your quick reply, Andrea! I temporarily turn the comp-speed to 0 and
less memories was required during the compilation. And the eln files are still
faster than elc files. Amazing works!
IMO, leim files may not need to be native-compiled. If you check those files,
you will found that they just some macros and tons of alists that map input codes
to characters. So they just need to be byte-compiled(to do the macro-expansion).
Another question is, polymorphic functions defined by `cl-defmethod` can't be
native compiled. `cl-defmethod` generates and byte-compile dynamic lambda to
support runtime multi-dispatch. I' m wondering is that any way to make
`cl-defmethod` compatible with native-comp, maybe Stefan Monnier have more ideas
on this.
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 19:14 ` Zhu Zihao
@ 2020-03-09 19:41 ` Andrea Corallo
2020-03-09 21:06 ` Stefan Monnier
1 sibling, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-09 19:41 UTC (permalink / raw)
To: Zhu Zihao; +Cc: Stefan Monnier, emacs-devel
Zhu Zihao <all_but_last@163.com> writes:
> Thanks for your quick reply, Andrea! I temporarily turn the comp-speed to 0 and
> less memories was required during the compilation. And the eln files are still
> faster than elc files. Amazing works!
>
> IMO, leim files may not need to be native-compiled. If you check those files,
> you will found that they just some macros and tons of alists that map input codes
> to characters. So they just need to be byte-compiled(to do the macro-expansion).
Yes I've seen it but the next question would be how to define such list
of "not to be compiled" files? I though was cleaner to implement
deferred compilation because this is more generic, but maybe a
not-compile list is also a good suggestion that is very easy to
implement in addition.
Thoughts?
> Another question is, polymorphic functions defined by `cl-defmethod` can't be
> native compiled. `cl-defmethod` generates and byte-compile dynamic lambda to
> support runtime multi-dispatch. I' m wondering is that any way to make
> `cl-defmethod` compatible with native-comp, maybe Stefan Monnier have more ideas
> on this.
I've never disassembled a cl method but generally speaking for now we
just do not native compile all lambdas :)
We are really in bring-up phase to be clear.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 19:14 ` Zhu Zihao
2020-03-09 19:41 ` Andrea Corallo
@ 2020-03-09 21:06 ` Stefan Monnier
1 sibling, 0 replies; 210+ messages in thread
From: Stefan Monnier @ 2020-03-09 21:06 UTC (permalink / raw)
To: Zhu Zihao; +Cc: emacs-devel, Andrea Corallo
> Another question is, polymorphic functions defined by `cl-defmethod` can't be
> native compiled. `cl-defmethod` generates and byte-compile dynamic lambda to
> support runtime multi-dispatch. I' m wondering is that any way to make
> `cl-defmethod` compatible with native-comp,
AFAIK it's not *incompatible*, it's just that the dispatch functions
generated by cl-defmethod stay as byte-compiled, because they're
dynamically generated rather than being written into a .elc file.
There's a lot of room for improvement in this area, but I think it will
inevitably require specific treatment. While the speed of execution of
those dispatch functions is important, the speed with which we can
(re)create them is also important, so a naive approach of passing it to
the native compiler to save into a temp file which we then load is
likely to be too costly in some cases.
Stefan
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 18:52 ` Andrea Corallo
2020-03-09 19:14 ` Zhu Zihao
@ 2020-03-09 19:18 ` Eli Zaretskii
2020-03-09 19:42 ` Andrea Corallo
2020-04-05 21:48 ` Andrea Corallo
1 sibling, 2 replies; 210+ messages in thread
From: Eli Zaretskii @ 2020-03-09 19:18 UTC (permalink / raw)
To: Andrea Corallo; +Cc: all_but_last, emacs-devel
> From: Andrea Corallo <akrl@sdf.org>
> Date: Mon, 09 Mar 2020 18:52:17 +0000
> Cc: emacs-devel@gnu.org
>
> I've never measured the max memory consumption of the while bootstrap
> but I usually count like 2GB for each compilation unit when optimizing.
That might mean 32-bit build of Emacs will fail to compile Lisp files.
Did you try building a 32-bit Emacs with native compilation of Lisp?
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 19:18 ` Eli Zaretskii
@ 2020-03-09 19:42 ` Andrea Corallo
2020-04-05 21:48 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-09 19:42 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: all_but_last, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
>> From: Andrea Corallo <akrl@sdf.org>
>> Date: Mon, 09 Mar 2020 18:52:17 +0000
>> Cc: emacs-devel@gnu.org
>>
>> I've never measured the max memory consumption of the while bootstrap
>> but I usually count like 2GB for each compilation unit when optimizing.
>
> That might mean 32-bit build of Emacs will fail to compile Lisp files.
> Did you try building a 32-bit Emacs with native compilation of Lisp?
Good point, never tried. No idea what GCC does in these cases.
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-09 19:18 ` Eli Zaretskii
2020-03-09 19:42 ` Andrea Corallo
@ 2020-04-05 21:48 ` Andrea Corallo
1 sibling, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-04-05 21:48 UTC (permalink / raw)
To: Eli Zaretskii; +Cc: all_but_last, emacs-devel
Eli Zaretskii <eliz@gnu.org> writes:
> That might mean 32-bit build of Emacs will fail to compile Lisp files.
> Did you try building a 32-bit Emacs with native compilation of Lisp?
To update on the 32bit subject.
With what I've pushed today is working for me on x86_32bit with and
without wide int.
I tried bootstrapping and compiling a full Emacs at speed 2 but I got an
error for virtual memory exhaustion.
I sucessfully compiled at speed 2 the image dumped files and all the
remaing at speed 0 tho.
I guess managing to tune GCC could help memory consumption too but I
think I'll look into it after having finished what is still missing.
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-06 17:51 ` Andrea Corallo
2020-03-09 17:55 ` Zhu Zihao
@ 2020-03-10 16:15 ` Zhu Zihao
2020-03-10 17:46 ` Andrea Corallo
1 sibling, 1 reply; 210+ messages in thread
From: Zhu Zihao @ 2020-03-10 16:15 UTC (permalink / raw)
To: Andrea Corallo; +Cc: Zhu Zihao, emacs-devel
I' ve read some code of your benchmark. And there's a simple way to optimize
these bench programs a littel bit.
For some program use cl-defstruct, you can put `(cl-declaim (optimize (speed
3)))` in the file. This instructment will tell Emacs byte-compiler to delete
runtime type check code (which is harmful in bench game) in struct accessor.
An example is
https://gitlab.com/koral/elisp-benchmarks/-/blob/master/benchmarks/nbody.el.
After adding instructment and byte-compilation, the bench result is
(benchmark-run (elb-nbody-entry))
;; => (28.996680579 107 15.392465563000002)
28s was taken in 1000000 times iteration. And I believe native-comp will get
better result.
--
Zihao
^ permalink raw reply [flat|nested] 210+ messages in thread
* Re: On elisp running native
2020-03-10 16:15 ` Zhu Zihao
@ 2020-03-10 17:46 ` Andrea Corallo
0 siblings, 0 replies; 210+ messages in thread
From: Andrea Corallo @ 2020-03-10 17:46 UTC (permalink / raw)
To: Zhu Zihao; +Cc: emacs-devel
Zhu Zihao <all_but_last@163.com> writes:
> I' ve read some code of your benchmark. And there's a simple way to optimize
> these bench programs a littel bit.
>
> For some program use cl-defstruct, you can put `(cl-declaim (optimize (speed
> 3)))` in the file. This instructment will tell Emacs byte-compiler to delete
> runtime type check code (which is harmful in bench game) in struct accessor.
>
> An example is
> https://gitlab.com/koral/elisp-benchmarks/-/blob/master/benchmarks/nbody.el.
>
> After adding instructment and byte-compilation, the bench result is
>
> (benchmark-run (elb-nbody-entry))
> ;; => (28.996680579 107 15.392465563000002)
>
> 28s was taken in 1000000 times iteration. And I believe native-comp will get
> better result.
Hi Zhu Zihao,
yes there are a lot of ways these benchmarks could be optimized but IMO
these are to have a (as much avg as possible) case to optimize for. We
have taken some of them from the bench game just to have a starting
point but we haven't spent any time to optimize them on purpose.
Perhaps the day we'll want to compare our implementation against others
we'll work on some optimized entry for the bench game.
Thanks!
Andrea
--
akrl@sdf.org
^ permalink raw reply [flat|nested] 210+ messages in thread
end of thread, other threads:[~2021-01-01 20:07 UTC | newest]
Thread overview: 210+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-11-28 6:54 On elisp running native Andrea Corallo
2019-11-28 11:31 ` Óscar Fuentes
2019-11-28 14:07 ` Andrea Corallo
2019-11-28 12:47 ` Lars Ingebrigtsen
2019-11-28 20:15 ` Andrea Corallo
2019-11-29 6:03 ` Lars Brinkhoff
2019-11-28 16:04 ` Stefan Monnier
2019-11-28 17:28 ` Andrea Corallo
2019-11-28 21:06 ` Stefan Monnier
2019-11-28 22:52 ` Andrea Corallo
2019-11-29 0:20 ` Eric Abrahamsen
2019-11-29 14:59 ` Andrea Corallo
2019-11-29 15:10 ` Eli Zaretskii
2019-11-29 15:30 ` Stefan Monnier
2019-12-01 10:34 ` Andrea Corallo
2019-12-01 15:27 ` Stefan Monnier
2019-11-29 16:43 ` Andrea Corallo
2019-11-29 19:26 ` Eli Zaretskii
2019-11-29 21:18 ` Andrea Corallo
2019-12-08 20:18 ` Andrea Corallo
2019-12-26 13:05 ` Andrea Corallo
2019-12-26 17:30 ` Eli Zaretskii
2019-12-26 18:14 ` Andrea Corallo
2019-12-26 20:37 ` Eli Zaretskii
2019-12-26 22:10 ` Andrea Corallo
2019-12-27 9:12 ` Eli Zaretskii
2019-12-27 10:46 ` Andrea Corallo
2019-12-27 14:03 ` Eli Zaretskii
2019-12-28 1:17 ` Richard Stallman
2019-12-28 7:50 ` Eli Zaretskii
2019-12-28 8:17 ` Andrea Corallo
2019-12-29 0:08 ` Richard Stallman
2019-12-29 9:29 ` Andrea Corallo
2019-12-28 13:07 ` Dmitry Gutov
2019-12-28 13:28 ` Eli Zaretskii
2019-12-28 19:56 ` Óscar Fuentes
2019-12-29 19:59 ` chad
2019-12-30 8:17 ` arthur miller
2019-12-30 14:09 ` Dmitry Gutov
2019-12-30 15:25 ` Eli Zaretskii
2019-12-30 22:03 ` arthur miller
2019-12-31 3:34 ` Eli Zaretskii
2019-12-30 13:56 ` Dmitry Gutov
2019-12-30 14:04 ` Juanma Barranquero
2019-12-30 8:47 ` Achim Gratz
2019-12-27 15:53 ` Stefan Monnier
2019-12-27 21:02 ` Andrea Corallo
2019-12-27 22:19 ` Stefan Monnier
2019-12-28 8:34 ` Eli Zaretskii
2019-12-28 8:56 ` Andrea Corallo
2019-12-28 10:06 ` Eli Zaretskii
2019-12-28 11:57 ` Andrea Corallo
2019-12-28 13:34 ` Eli Zaretskii
2019-12-28 14:35 ` Andrea Corallo
2019-12-28 18:38 ` Eli Zaretskii
2019-12-28 22:13 ` Andrea Corallo
2019-12-31 9:55 ` Andrea Corallo
2019-12-31 15:49 ` Eli Zaretskii
2020-02-24 23:21 ` Andrea Corallo
2020-02-25 6:14 ` yyoncho
2020-02-25 7:29 ` Eli Zaretskii
2020-02-25 12:59 ` Andrea Corallo
2020-02-28 4:34 ` Eli Zaretskii
2020-02-28 8:26 ` Andrea Corallo
2020-02-28 9:23 ` yyoncho
2020-04-26 12:00 ` Andrea Corallo
2020-04-27 4:41 ` yyoncho
2019-12-26 20:26 ` Stefan Monnier
2019-12-26 20:58 ` dick.r.chiang
2019-12-26 21:57 ` Andrea Corallo
2019-12-27 8:22 ` Eli Zaretskii
2019-12-27 10:49 ` Andrea Corallo
2019-12-28 16:10 ` dick.r.chiang
2019-12-28 17:31 ` Eli Zaretskii
2019-12-31 10:24 ` Andrea Corallo
2019-12-31 15:51 ` Eli Zaretskii
2019-12-31 16:21 ` Andrea Corallo
2019-12-31 16:47 ` Eli Zaretskii
2019-12-31 22:15 ` Óscar Fuentes
2020-01-01 12:41 ` Andrea Corallo
2020-01-01 15:14 ` Stefan Monnier
2020-01-01 18:42 ` Andrea Corallo
2020-01-02 7:55 ` arthur miller
2020-01-02 11:31 ` Andrea Corallo
2020-01-02 13:53 ` Eli Zaretskii
2020-01-04 18:17 ` Stefan Monnier
2020-01-05 1:33 ` Jean-Christophe Helary
2020-01-05 4:51 ` Stefan Monnier
2020-01-05 5:18 ` Jean-Christophe Helary
2020-01-05 6:36 ` Stefan Monnier
2020-01-02 13:47 ` Eli Zaretskii
2020-01-04 18:14 ` Stefan Monnier
2020-01-05 10:00 ` Andrea Corallo
2020-01-02 11:47 ` Andrea Corallo
2020-01-02 16:52 ` Ergus
2020-01-02 17:55 ` Andrea Corallo
2020-01-11 14:10 ` Andrea Corallo
2020-02-04 11:28 ` Andrea Corallo
2020-02-04 15:30 ` Eli Zaretskii
2020-02-04 18:39 ` Andrea Corallo
2020-02-04 18:58 ` Eli Zaretskii
2020-02-04 19:00 ` Paul Eggert
2020-02-04 20:30 ` Andrea Corallo
2020-02-05 10:15 ` Andrea Corallo
2020-02-05 14:24 ` Stefan Monnier
2020-02-15 9:29 ` Andrea Corallo
2020-02-16 18:18 ` Andrea Corallo
2020-02-22 10:15 ` On elisp running native - deferred compilation Andrea Corallo
2020-02-22 14:19 ` Stefan Monnier
2020-03-04 19:34 ` On elisp running native Andrea Corallo
2020-03-04 20:21 ` Michael Albinus
2020-03-04 22:16 ` Andrea Corallo
2020-03-05 19:33 ` Michael Albinus
2020-03-06 21:44 ` Andrea Corallo
2020-03-07 9:09 ` Michael Albinus
2020-03-07 9:33 ` Andrea Corallo
2020-03-07 9:41 ` Michael Albinus
[not found] ` <xjf36akjx8e.fsf@sdf.org>
[not found] ` <87y2sca2ex.fsf@gmx.de>
[not found] ` <xjfy2scigxn.fsf@sdf.org>
[not found] ` <xjftv30hytf.fsf@sdf.org>
2020-03-07 17:57 ` Michael Albinus
2020-03-07 18:49 ` Andrea Corallo
2020-03-07 18:54 ` Michael Albinus
2020-03-07 19:19 ` Andrea Corallo
2020-03-08 15:03 ` Andrea Corallo
2020-03-08 20:16 ` Michael Albinus
2020-03-08 20:55 ` Andrea Corallo
2020-03-08 21:09 ` Michael Albinus
2020-03-08 21:29 ` Andrea Corallo
2020-03-08 21:32 ` Michael Albinus
2020-03-08 21:37 ` Andrea Corallo
2020-03-05 10:54 ` Adam Porter
2020-03-05 12:43 ` Gong-Yi Liao 廖宮毅
2020-03-05 13:27 ` Andrea Corallo
2020-03-08 14:37 ` Gong-Yi Liao 廖宮毅
2020-03-08 15:13 ` Andrea Corallo
2020-03-15 19:00 ` Gong-Yi Liao 廖宮毅
2020-03-15 20:00 ` Andrea Corallo
2020-03-05 13:19 ` Andrea Corallo
2020-03-05 14:42 ` Adam Porter
2020-03-05 19:46 ` Andrea Corallo
2020-03-06 21:15 ` Adam Porter
2020-03-06 21:23 ` Adam Porter
2020-03-06 21:52 ` Andrea Corallo
2020-03-11 22:04 ` Andrea Corallo
2020-03-12 17:10 ` Adam Porter
2020-03-12 17:34 ` Stefan Monnier
2020-03-12 18:03 ` Andrea Corallo
2020-03-23 21:36 ` On elisp running native - Update7 Andrea Corallo
2020-03-24 2:59 ` Gong-Yi Liao 廖宮毅
2020-03-24 8:44 ` Andrea Corallo
2020-05-16 15:24 ` On elisp running native - Update 8 Andrea Corallo
2020-05-16 16:21 ` Eli Zaretskii
2020-05-16 20:44 ` Andrea Corallo
2020-05-16 18:22 ` Gregor Zattler
2020-05-16 19:59 ` Andrea Corallo
2020-05-17 5:26 ` Gregor Zattler
2020-06-14 13:23 ` On elisp running native - Update 9 Andrea Corallo
2020-06-14 17:23 ` arthur miller
2020-06-15 5:04 ` Gong-Yi Liao 廖宮毅
2020-06-15 7:41 ` Andrea Corallo
2020-06-17 23:54 ` Stefan Monnier
2020-06-18 6:55 ` Andrea Corallo
2020-07-16 20:00 ` On elisp running native - Update 10 Andrea Corallo via Emacs development discussions.
2020-08-17 18:43 ` On elisp running native - Update 11 Andrea Corallo via Emacs development discussions.
2020-08-18 5:13 ` Arthur Miller
2020-08-18 8:17 ` Andrea Corallo via Emacs development discussions.
2020-09-26 10:35 ` On elisp running native - Update 12 Andrea Corallo via Emacs development discussions.
2020-09-26 14:29 ` T.V Raman
2020-09-26 14:56 ` Andrea Corallo via Emacs development discussions.
2020-09-26 15:07 ` T.V Raman
2020-09-28 12:22 ` Lars Brinkhoff
2020-09-28 14:03 ` T.V Raman
2020-09-27 10:35 ` Arthur Miller
2020-09-27 18:28 ` Andrea Corallo via Emacs development discussions.
2020-09-27 18:30 ` Lars Ingebrigtsen
2020-09-27 20:39 ` T.V Raman
2020-09-27 22:10 ` Stefan Monnier
2020-09-28 6:06 ` Eli Zaretskii
2020-09-28 10:28 ` Andrea Corallo via Emacs development discussions.
2020-09-28 10:45 ` Eli Zaretskii
2020-09-28 11:02 ` Andrea Corallo via Emacs development discussions.
2020-09-27 20:37 ` T.V Raman
2020-09-28 14:10 ` Yuan Fu
2020-09-28 14:51 ` Andrea Corallo via Emacs development discussions.
2020-09-28 15:20 ` Yuan Fu
2020-09-28 15:32 ` Andrea Corallo via Emacs development discussions.
2020-09-28 15:19 ` Caio Henrique
2020-09-28 15:39 ` Andrea Corallo via Emacs development discussions.
2020-09-28 16:12 ` Caio Henrique
2020-09-28 17:21 ` Andrea Corallo via Emacs development discussions.
2021-01-01 18:37 ` On elisp running native - Update 13 Andrea Corallo via Emacs development discussions.
2021-01-01 20:07 ` tomas
2020-01-01 15:19 ` On elisp running native Óscar Fuentes
2020-01-01 18:48 ` Andrea Corallo
2020-01-01 14:59 ` Stefan Monnier
2020-03-15 14:21 ` Spenser Truex
2020-03-15 20:10 ` Andrea Corallo
2020-03-18 14:58 ` Lars Brinkhoff
2020-03-18 17:14 ` Andrea Corallo
-- strict thread matches above, loose matches on Subject: below --
2020-01-02 21:47 arthur miller
2020-03-06 13:28 Zhu Zihao
2020-03-06 17:51 ` Andrea Corallo
2020-03-09 17:55 ` Zhu Zihao
2020-03-09 18:52 ` Andrea Corallo
2020-03-09 19:14 ` Zhu Zihao
2020-03-09 19:41 ` Andrea Corallo
2020-03-09 21:06 ` Stefan Monnier
2020-03-09 19:18 ` Eli Zaretskii
2020-03-09 19:42 ` Andrea Corallo
2020-04-05 21:48 ` Andrea Corallo
2020-03-10 16:15 ` Zhu Zihao
2020-03-10 17:46 ` Andrea Corallo
Code repositories for project(s) associated with this public inbox
https://git.savannah.gnu.org/cgit/emacs.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).