Re: [ELPA] New package: elisp-benchmarks

[Andrea Corallo <akrl@sdf.org>]

[-- Attachment #1: Type: text/plain, Size: 98 bytes --]

Hi, I attach the patch git formatted.  I've also fixed two nits.

Bests

Andrea

-- 
akrl@sdf.org

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: 0001-elisp-benckmarks-New-package-initial-add.patch --]
[-- Type: text/x-patch, Size: 19686 bytes --]

From 81b3e346dc4c9e612bf74fb0f29bf9f170cf6947 Mon Sep 17 00:00:00 2001
From: Andrea Corallo <akrl@sdf.org>
Date: Sun, 1 Dec 2019 17:26:37 +0100
Subject: [PATCH] [elisp-benckmarks] New package initial add

Add a package for benchmarking the elisp implementation.
---
 .../benchmarks/bubble-no-cons.el              |  45 ++++++
 .../elisp-benchmarks/benchmarks/bubble.el     |  48 ++++++
 .../elisp-benchmarks/benchmarks/fibn-rec.el   |  33 ++++
 .../elisp-benchmarks/benchmarks/fibn-tc.el    |  35 ++++
 packages/elisp-benchmarks/benchmarks/fibn.el  |  40 +++++
 .../elisp-benchmarks/benchmarks/inclist.el    |  42 +++++
 .../elisp-benchmarks/benchmarks/listlen-tc.el |  41 +++++
 .../elisp-benchmarks/benchmarks/pidigits.el   |  71 +++++++++
 packages/elisp-benchmarks/elisp-benchmarks.el | 150 ++++++++++++++++++
 9 files changed, 505 insertions(+)
 create mode 100644 packages/elisp-benchmarks/benchmarks/bubble-no-cons.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/bubble.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/fibn-rec.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/fibn-tc.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/fibn.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/inclist.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/listlen-tc.el
 create mode 100644 packages/elisp-benchmarks/benchmarks/pidigits.el
 create mode 100644 packages/elisp-benchmarks/elisp-benchmarks.el

diff --git a/packages/elisp-benchmarks/benchmarks/bubble-no-cons.el b/packages/elisp-benchmarks/benchmarks/bubble-no-cons.el
new file mode 100644
index 000000000..012a32ce5
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/bubble-no-cons.el
@@ -0,0 +1,45 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Like bubble but in place.
+
+(require 'cl-lib)
+
+(defvar elb-bubble-len 1000)
+(defvar elb-bubble-list (mapcar #'random (make-list elb-bubble-len
+						    most-positive-fixnum)))
+(defun elb-bubble-no-cons (list)
+  (cl-loop repeat (length list)
+	   do
+	   (cl-loop for x on list
+		    for a = (car x)
+		    for b = (cadr x)
+		    when (and b (> a b))
+		      do (setcar x b)
+		         (setcar (cdr x) a)
+		    finally (return list))))
+
+(defun elb-bubble-no-cons-entry ()
+  (cl-loop repeat 200
+	   for l = (copy-sequence elb-bubble-list)
+	   do (elb-bubble-no-cons l)))
+
+(provide 'elb-bubble)
diff --git a/packages/elisp-benchmarks/benchmarks/bubble.el b/packages/elisp-benchmarks/benchmarks/bubble.el
new file mode 100644
index 000000000..d7101b1b9
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/bubble.el
@@ -0,0 +1,48 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; From:
+;; https://www.emacswiki.org/emacs/EmacsLispBenchmark
+
+(require 'cl-lib)
+
+(defvar elb-bubble-len 1000)
+(defvar elb-bubble-list (mapcar #'random (make-list elb-bubble-len
+						    most-positive-fixnum)))
+
+(defun elb-bubble (list)
+  (let ((i (length list)))
+    (while (> i 1)
+      (let ((b list))
+        (while (cdr b)
+          (when (< (cadr b) (car b))
+            (setcar b (prog1 (cadr b)
+                        (setcdr b (cons (car b) (cddr b))))))
+          (setq b (cdr b))))
+      (setq i (1- i)))
+    list))
+
+(defun elb-bubble-entry ()
+  (cl-loop repeat 100
+	   for l = (copy-sequence elb-bubble-list)
+	   do (elb-bubble l)))
+
+(provide 'elb-bubble)
diff --git a/packages/elisp-benchmarks/benchmarks/fibn-rec.el b/packages/elisp-benchmarks/benchmarks/fibn-rec.el
new file mode 100644
index 000000000..a8e4b6c96
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/fibn-rec.el
@@ -0,0 +1,33 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Fibonacci sequence recursive algo.
+
+(defun elb-fib (n)
+  (cond ((= n 0) 0)
+	((= n 1) 1)
+	(t (+ (elb-fib (- n 1))
+	      (elb-fib (- n 2))))))
+
+(defun elb-fibn-rec-entry ()
+  (elb-fib 37))
+
+(provide 'fibn-rec)
diff --git a/packages/elisp-benchmarks/benchmarks/fibn-tc.el b/packages/elisp-benchmarks/benchmarks/fibn-tc.el
new file mode 100644
index 000000000..83d571bee
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/fibn-tc.el
@@ -0,0 +1,35 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Fibonacci sequence tail recursive algo.
+
+(require 'cl-lib)
+
+(defun elb-fibn-tc (a b count)
+  (if (= count 0)
+      b
+    (elb-fibn-tc (+ a b) a (- count 1))))
+
+(defun elb-fibn-tc-entry ()
+  (cl-loop repeat 1000000
+	   do (elb-fibn-tc 1 0 80)))
+
+(provide 'fibn-tc)
diff --git a/packages/elisp-benchmarks/benchmarks/fibn.el b/packages/elisp-benchmarks/benchmarks/fibn.el
new file mode 100644
index 000000000..af5347760
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/fibn.el
@@ -0,0 +1,40 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Adapted to elisp from CL version from:
+;; https://drmeister.wordpress.com/2015/07/30/timing-data-comparing-cclasp-to-c-sbcl-and-python/
+
+(defun elb-fibn (reps num)
+  (let ((z 0))
+    (dotimes (_ reps)
+      (let ((p1 1)
+            (p2 1))
+        (dotimes (_ (- num 2))
+          (setf z (+ p1 p2)
+                p2 p1
+                p1 z))))
+    z))
+
+(defun elb-fibn-entry ()
+  ;; Use 80 to stay in the fixnum range.
+  (elb-fibn 3000000 80))
+
+(provide 'elb-fibn)
diff --git a/packages/elisp-benchmarks/benchmarks/inclist.el b/packages/elisp-benchmarks/benchmarks/inclist.el
new file mode 100644
index 000000000..63837cbfa
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/inclist.el
@@ -0,0 +1,42 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Iteratively increment the elements of a list.
+
+(require 'cl-lib)
+
+(defvar elb-inclist-no-type-hints-len 50000)
+(defvar elb-inclist-no-type-hints-list
+  (mapcar #'random (make-list elb-inclist-no-type-hints-len 100)))
+
+(defun elb-inclist (l)
+  (prog1 l
+    (while l
+      (let ((c l))
+	(cl-incf (car c))
+	(setq l (cdr c))))))
+
+(defun elb-inclist-entry ()
+  (let ((l (copy-sequence elb-inclist-no-type-hints-list)))
+    (cl-loop repeat 10000
+	     do (elb-inclist l))))
+
+(provide 'elb-inclist)
diff --git a/packages/elisp-benchmarks/benchmarks/listlen-tc.el b/packages/elisp-benchmarks/benchmarks/listlen-tc.el
new file mode 100644
index 000000000..02327e512
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/listlen-tc.el
@@ -0,0 +1,41 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Compute the length of a list tail recursively.
+
+(require 'cl-lib)
+
+(defvar elb-listlen-tc-len 300)
+(defvar elb-listlen-tc-list
+  (mapcar #'random (make-list elb-listlen-tc-len 100)))
+
+(defun elb-listlen-tc (l n)
+  (if (null l)
+      n
+    (cl-incf n)
+    (elb-listlen-tc (cdr l) n)))
+
+(defun elb-listlen-tc-entry ()
+  (let ((l (copy-sequence elb-listlen-tc-list)))
+    (cl-loop repeat 350000
+	     do (elb-listlen-tc l 0))))
+
+(provide 'listlen-tc)
diff --git a/packages/elisp-benchmarks/benchmarks/pidigits.el b/packages/elisp-benchmarks/benchmarks/pidigits.el
new file mode 100644
index 000000000..de1eb42a3
--- /dev/null
+++ b/packages/elisp-benchmarks/benchmarks/pidigits.el
@@ -0,0 +1,71 @@
+;; -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2019 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Adapted to elisp from CL version from:
+;; https://salsa.debian.org/benchmarksgame-team/benchmarksgame/
+
+;; Compute pi
+
+(require 'cl-lib)
+
+(defvar elb-acc)
+(defvar elb-den)
+(defvar elb-num)
+
+(defun elb-extract-digit (nth)
+  (truncate (+ (* elb-num nth) elb-acc) elb-den))
+
+(defun elb-eliminate-digit (d)
+  (cl-decf elb-acc (* elb-den d))
+  (setf elb-acc (* elb-acc 10)
+	elb-num (* elb-num 10)))
+
+(defun elb-next-term (k)
+  (let ((k2 (1+ (* k 2))))
+    (cl-incf elb-acc (* elb-num 2))
+    (setf elb-acc (* elb-acc k2)
+	  elb-den (* elb-den k2)
+	  elb-num (* elb-num k))))
+
+(defun elb-pidigits (x)
+  (let ((elb-acc 0)
+	(elb-den 1)
+	(elb-num 1)
+	(res ()))
+    (cl-do ((d 0) (k 0) (i 0) (n 10000))
+	((>= i n))
+      (setf n x)
+      (elb-next-term (cl-incf k))
+      (unless (> elb-num elb-acc)
+	(setf d (elb-extract-digit 3))
+	(unless (/= d (elb-extract-digit 4))
+	  (push d res)
+	  (cl-incf i)
+ 	  ;; (when (= (mod (cl-incf i) 10) 0)
+	  ;;   (message "%d" i))
+	  (elb-eliminate-digit d))))
+    (reverse res)))
+
+(defun elb-pidigits-entry ()
+  (cl-loop repeat 1000
+	   do (elb-pidigits 500)))
+
+(provide 'elb-pidigits)
diff --git a/packages/elisp-benchmarks/elisp-benchmarks.el b/packages/elisp-benchmarks/elisp-benchmarks.el
new file mode 100644
index 000000000..1b95ab6f7
--- /dev/null
+++ b/packages/elisp-benchmarks/elisp-benchmarks.el
@@ -0,0 +1,150 @@
+;;; elisp-benchmarks.el --- elisp benchamrks collection -*- lexical-binding:t -*-
+
+;; Copyright (C) 2019  Free Software Foundation, Inc.
+
+;; Author: akrl@sdf.org
+;; Maintainer: akrl@sdf.org
+;; Version: 1.0
+;; Keywords: languages, lisp
+;; Created: 2019-01-12
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;; In use for testing the Emacs Lisp implementation performance.
+
+;; To minimize CPU frequency bouncing effects and other sources of
+;; noise all benchmarks are repeated `elb-runs' times by default.
+
+;; To add a new benchmark just depose the file into the benchmarks/
+;; directory.  Every benchmark foo.el has to define as entry-point a
+;; function foo-entry.
+
+;; Tests are of an arbitrary length that on my machine is in the
+;; magnitude order of the 10 seconds for each single run
+;; byte-compiled.  Please consider this as relative measure when
+;; adding new benchmarks.
+
+;;; Usage:
+;; emacs -batch -l .../elisp-benchmarks.el -f elisp-benchmarks-run
+
+;;; Code:
+
+(require 'cl-lib)
+(require 'benchmark)
+(require 'outline)
+(require 'org)
+
+(defgroup elb nil
+  "Emacs Lisp benchmarks."
+  :group 'lisp)
+
+(defcustom elb-runs 3
+  "Total number of benchmark iterations."
+  :type 'number
+  :group 'comp)
+
+(defconst elb-bench-directory
+  (concat (file-name-directory (or load-file-name buffer-file-name))
+	  "benchmarks/"))
+
+(defconst elb-result-buffer-name "elisp-benchmarks-results"
+  "Buffer name where results are presented.")
+
+(defun elb-std-deviation (list)
+  "Return the standard deviation of the elements in LIST."
+  (let* ((n (length list))
+	 (mean (/ (cl-loop for x in list
+			   sum x)
+		  n)))
+    (sqrt (/ (cl-loop for x in list
+		   sum (expt (- x mean) 2))
+	  (1- n)))))
+
+;;;###autoload
+(defun elisp-benchmarks-run (&optional selector recompile runs)
+  "Run all the benchmarks and present the results.
+If non nil SELECTOR is a regexp to match the benchmark names to be executed.
+The test is repeated RUNS number of times.  If RUNS is nil `elb-runs' is used as
+default.
+RECOMPILE all the benchmark folder when non nil."
+  (interactive)
+  (cl-loop with runs = (or runs elb-runs)
+	   repeat runs
+	   for i from 1
+	   named test-loop
+	   with res = (make-hash-table :test #'equal)
+	   with sources = (directory-files elb-bench-directory t "\\.el$")
+	   with tests = (if selector
+			    (cl-loop for f in sources
+				     when (string-match selector f)
+				       collect (file-name-base f))
+			  (mapcar #'file-name-base sources))
+	   initially
+	   (if recompile
+	       (mapc (lambda (f) (byte-compile-file f t)) sources)
+	     (mapc #'load (mapcar #'file-name-sans-extension sources)))
+	   (cl-loop for test in tests
+		    do (puthash test () res))
+	   do
+	   (message "Iteration number: %d" i)
+	   (cl-loop for test in tests
+		    for entry-point = (intern (concat "elb-" test "-entry"))
+		    do
+		    (garbage-collect)
+		    (message "Running %s..." test)
+		    (push (eval `(benchmark-run nil (,entry-point)) t)
+			  (gethash test res)))
+	   finally
+	   (pop-to-buffer elb-result-buffer-name)
+	   (erase-buffer)
+	   (insert "* Results\n\n")
+	   (insert "  |test|non-gc avg (s)|gc avg (s)|gcs avg|tot avg (s)|tot avg err (s)\n")
+	   (insert "|-\n")
+	   (cl-loop for test in tests
+		    for l = (gethash test res)
+		    for test-elapsed = (cl-loop for x in l sum (car x))
+		    for test-gcs = (cl-loop for x in l sum (cadr x))
+		    for test-gc-elapsed = (cl-loop for x in l sum (caddr x))
+		    for test-err = (elb-std-deviation (mapcar #'car l))
+		    do
+		    (insert (apply #'format "|%s|%.2f|%.2f|%d|%.2f" test
+				   (mapcar (lambda (x) (/ x runs))
+					   (list (- test-elapsed test-gc-elapsed)
+						 test-gc-elapsed test-gcs
+						 test-elapsed))))
+		    (insert (format "|%.2f\n" test-err))
+		    summing test-elapsed into elapsed
+		    summing test-gcs into gcs
+		    summing test-gc-elapsed into gc-elapsed
+		    collect test-err into errs
+		    finally
+		    (insert "|-\n")
+		    (insert (apply #'format "|total|%.2f|%.2f|%d|%.2f"
+				   (mapcar (lambda (x) (/ x runs))
+					   (list (- elapsed gc-elapsed)
+						 gc-elapsed gcs elapsed))))
+		    (insert (format "|%.2f\n"
+				    (sqrt (apply #'+ (mapcar (lambda (x)
+							    (expt x 2))
+							  errs))))))
+	   (org-table-align)
+	   (goto-char (point-min))
+	   (if noninteractive
+	       (message (buffer-string))
+	     (org-mode)
+	     (outline-show-subtree))))
+
+(provide 'elisp-benchmarks)
+;;; elisp-benchmarks.el ends here
-- 
2.17.1