=== modified file '.bzrignore' --- .bzrignore 2013-06-22 02:41:14 +0000 +++ .bzrignore 2013-08-13 11:08:17 +0000 @@ -172,6 +172,8 @@ src/stamp-h.in src/temacs test/indent/*.new +test/automated/*.log +test/automated/failure-tests +* src/globals.h src/gl-stamp === modified file 'ChangeLog' --- ChangeLog 2013-07-13 00:01:43 +0000 +++ ChangeLog 2013-08-13 11:08:33 +0000 @@ -1,3 +1,8 @@ +2013-08-13 Kenichi Handa + + * .bzrignore: Add test/automated/*.log and + test/automated/failure-tests. + 2013-07-13 Paul Eggert * configure.ac: Simplify --with-file-notification handling. === modified file 'lisp/ChangeLog' --- lisp/ChangeLog 2013-07-20 11:51:53 +0000 +++ lisp/ChangeLog 2013-08-13 09:18:54 +0000 @@ -1,3 +1,8 @@ +2013-08-13 Kenichi Handa + + * emacs-lisp/ert.el (ert-run-tests-batch-and-exit-single) + (ert-summary-report): New functions. + 2013-07-20 Kenichi Handa * international/mule.el (coding-system-iso-2022-flags): Add === modified file 'lisp/emacs-lisp/ert.el' --- lisp/emacs-lisp/ert.el 2013-07-11 16:13:38 +0000 +++ lisp/emacs-lisp/ert.el 2013-08-13 10:54:58 +0000 @@ -31,7 +31,9 @@ ;; `defun' but defines a test, and `ert-run-tests-interactively', ;; which runs tests and offers an interactive interface for inspecting ;; results and debugging. There is also -;; `ert-run-tests-batch-and-exit' for non-interactive use. +;; `ert-run-tests-batch-and-exit', +;; `ert-run-tests-batch-and-exit-single', and `ert-summary-report' for +;; non-interactive use. ;; ;; The body of `ert-deftest' forms resembles a function body, but the ;; additional operators `should', `should-not' and `should-error' are @@ -1409,6 +1411,93 @@ (backtrace)) (kill-emacs 2)))) +;;;###autoload +(defun ert-run-tests-batch-and-exit-single () + "Like `ert-run-tests-batch-and-exit', but run tests in a single file." + ;; It is intended that this function is called in batch mode with a + ;; test file as the argument and stderr redirected to a log file. A + ;; specially formated line is written at the end of that log file. + ;; That line is parsed by `ert-summary-report' later. + (let* ((test-file (pop command-line-args-left)) + (compiled (concat (file-name-sans-extension test-file) ".elc")) + (base (file-name-nondirectory test-file)) + ;; Some tests (e.g. flymake-tests.el) calls kill-buffer which + ;; may lead to process-kill-buffer-query-function requiring + ;; user iteraction. Bind this variable to nil avoid that + ;; problem. + (kill-buffer-query-functions nil) + (prefix (format "Running tests in %s..." base))) + (unwind-protect + (progn + ;; Load a byte-compiled one or TEST-FILE itself. + (if (file-newer-than-file-p compiled test-file) + (progn + (setq base (file-name-nondirectory compiled)) + (load-file compiled)) + (let ((buf (find-file-noselect test-file))) + (if (with-current-buffer buf + (and (boundp 'no-byte-compile) no-byte-compile)) + (with-current-buffer buf + (eval-buffer)) + (if (byte-compile-file test-file t) + (setq base (file-name-nondirectory compiled)) + (princ (format "%s failed to compile the file\n" prefix)) + (message "##REPORT:(compile-error \"%s\")##" base) + (kill-emacs 0))))) + ;; Run tests. + (let* ((stats (ert-run-tests-batch)) + (total (ert-stats-total stats)) + (expected (ert-stats-completed-expected stats))) + (if (= total expected) + (princ (format "%s passed all %d tests\n" prefix total)) + (princ (format "%s passed %d tests out of %d\n" + prefix expected total))) + (message "##REPORT:(done %d %d)##" total expected) + (kill-emacs 0))) + (princ (format "%s failed to load the file\n" prefix)) + (message "##REPORT:(load-error \"%s\")##" base) + (kill-emacs 0)))) + +;;;###autoload +(defun ert-summary-report () + "Print a test report summarizing test log files." + ;; It is intended that the function is called after all log files + ;; corresponding to test files were generated by + ;; ert-run-tests-batch-and-exit-single. + (let ((result (cons 0 0)) + errors) + (with-temp-buffer + (dolist (logfile command-line-args-left) + (erase-buffer) + (insert-file-contents logfile) + (goto-char (point-max)) + (or (search-backward "##REPORT:" nil t) + (error "no report line: %s" logfile)) + (goto-char (match-end 0)) + (let ((form (read (current-buffer)))) + (if (eq (car form) 'done) + (progn + (setcar result (+ (car result) (nth 1 form))) + (setcdr result (+ (cdr result) (nth 2 form)))) + (push form errors))))) + (setq command-line-args-left nil) + (message " ## Summary ##") + (message " Ran %s tests, %d results as expected, %d unexpected" + (car result) (cdr result) (- (car result) (cdr result))) + (when errors + (message "\n Following test files have problems:") + (dolist (err errors) + (let* ((tag (car err)) + (reason (cond ((eq tag 'emacs-error) + "emacs aborted while processing this file") + ((eq tag 'load-error) + "loading this file failed") + ((eq tag 'compile-error) + "compiling this file failed") + (t + "unknown error")))) + (message " %s: %s" (nth 1 err) reason))))) + (kill-emacs 0)) ;;; Utility functions for load/unload actions. === modified file 'test/ChangeLog' --- test/ChangeLog 2013-07-13 01:55:58 +0000 +++ test/ChangeLog 2013-08-13 11:05:23 +0000 @@ -1,3 +1,11 @@ +2013-08-13 Kenichi Handa + + * automated/Makefile.in (.SUFFIXES): Add .log. + (TEST_LOGS, FAILURE_TESTS, FAILURE_LOGS): New variables. + (.el.log, parallel, failure-tests/compile-error.el) + (failure-tests/load-error.el, failure-tests/emacs-error.el) + (parallel-with-failure-test, clean): New targets. + 2013-07-13 Fabián Ezequiel Gallina * automated/python-tests.el (python-imenu-create-index-2) === modified file 'test/automated/Makefile.in' --- test/automated/Makefile.in 2013-06-27 09:26:54 +0000 +++ test/automated/Makefile.in 2013-08-13 06:17:51 +0000 @@ -67,7 +67,7 @@ # subdirectories, to make sure require's and load's in the files being # compiled find the right files. -.SUFFIXES: .elc .el +.SUFFIXES: .elc .el .log # An old-fashioned suffix rule, which, according to the GNU Make manual, # cannot have prerequisites. @@ -136,6 +136,41 @@ cd $(test); rm -f *.elc */*.elc */*/*.elc */*/*/*.elc $(MAKE) $(MFLAGS) compile EMACS=$(EMACS) +TEST_LOGS = $(patsubst %.el, %.log, $(wildcard $(test)/*.el)) + +.el.log: + @cd $(test); \ + ($(emacs) -f ert-run-tests-batch-and-exit-single $< 2> $@ || \ + (BASE="`basename $<`"; \ + echo "Running tests in $$BASE... fatal error"; \ + echo "##REPORT:(emacs-error \"$$BASE\")###" > $@)) + +parallel: $(TEST_LOGS) + @cd $(test); $(emacs) -f ert-summary-report $(TEST_LOGS) + +FAILURE_TESTS = \ + failure-tests/compile-error.el \ + failure-tests/load-error.el \ + failure-tests/emacs-error.el + +FAILURE_LOGS = $(patsubst %.el, %.log, $(FAILURE_TESTS)) + +failure-tests/compile-error.el: $(lastword $(MAKEFILE_LIST)) + @test -d `dirname "$@"` || mkdir `dirname "$@"` + @echo '(ert-deftest test-run-error (should kkk))' > $@ +failure-tests/load-error.el: $(lastword $(MAKEFILE_LIST)) + @test -d `dirname "$@"` || mkdir `dirname "$@"` + @echo 'kkk' > $@ +failure-tests/emacs-error.el: $(lastword $(MAKEFILE_LIST)) + @test -d `dirname "$@"` || mkdir `dirname "$@"` + @echo '(kill-emacs 1)' > $@ + +parallel-with-failure-test: $(TEST_LOGS) $(FAILURE_LOGS) + @cd $(test); $(emacs) -f ert-summary-report $(TEST_LOGS) $(FAILURE_LOGS) + +clean: + cd $(test); rm -f $(TEST_LOGS) $(FAILURE_TESTS) + bootstrap-clean: cd $(test); rm -f *.elc */*.elc */*/*.elc */*/*/*.elc