initial attempt at performance testing suite for notmuch.

unofficial mirror of notmuch@notmuchmail.org
 help / color / mirror / code / Atom feed

* initial attempt at performance testing suite for notmuch.
@ 2012-11-17 21:37 david
  2012-11-17 21:37 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
                   ` (2 more replies)
  0 siblings, 3 replies; 5+ messages in thread
From: david @ 2012-11-17 21:37 UTC (permalink / raw)
  To: notmuch

For a long time we have needed some commone set of tests to measure
changes to e.g. tagging and dump/restore. A non-trivial corpus of mail
messages is a bit unwieldy to be shipped with the source, so there is
a seperate tarball.  Currently the tarball for this needs to manually
uploaded to the notmuchmail.org http server; I have set up the first
version.

^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh
  2012-11-17 21:37 initial attempt at performance testing suite for notmuch david
@ 2012-11-17 21:37 ` david
  2012-11-17 21:37 ` [PATCH 2/2] test: initial performance testing infrastructure david
  2012-11-20  2:23 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
  2 siblings, 0 replies; 5+ messages in thread
From: david @ 2012-11-17 21:37 UTC (permalink / raw)
  To: notmuch; +Cc: David Bremner

From: David Bremner <bremner@debian.org>

The idea is to use some of the simpler parts of the test suite
infrastructure to help run performance tests.
---
 test/test-lib-common.sh |  147 +++++++++++++++++++++++++++++++++++++++++++++++
 test/test-lib.sh        |  129 +----------------------------------------
 2 files changed, 148 insertions(+), 128 deletions(-)
 create mode 100644 test/test-lib-common.sh

diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh
new file mode 100644
index 0000000..e1eaa5a
--- /dev/null
+++ b/test/test-lib-common.sh
@@ -0,0 +1,147 @@
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see http://www.gnu.org/licenses/ .
+
+# This file contains common code to be used by both the regular
+# (correctness) tests and the performance tests.
+
+find_notmuch_path ()
+{
+    dir="$1"
+
+    while [ -n "$dir" ]; do
+	bin="$dir/notmuch"
+	if [ -x "$bin" ]; then
+	    echo "$dir"
+	    return
+	fi
+	dir="$(dirname "$dir")"
+	if [ "$dir" = "/" ]; then
+	    break
+	fi
+    done
+}
+
+# Test the binaries we have just built.  The tests are kept in
+# test/ subdirectory and are run in 'trash directory' subdirectory.
+TEST_DIRECTORY=$(pwd)
+notmuch_path=`find_notmuch_path "$TEST_DIRECTORY"`
+if test -n "$valgrind"
+then
+	make_symlink () {
+		test -h "$2" &&
+		test "$1" = "$(readlink "$2")" || {
+			# be super paranoid
+			if mkdir "$2".lock
+			then
+				rm -f "$2" &&
+				ln -s "$1" "$2" &&
+				rm -r "$2".lock
+			else
+				while test -d "$2".lock
+				do
+					say "Waiting for lock on $2."
+					sleep 1
+				done
+			fi
+		}
+	}
+
+	make_valgrind_symlink () {
+		# handle only executables
+		test -x "$1" || return
+
+		base=$(basename "$1")
+		symlink_target=$TEST_DIRECTORY/../$base
+		# do not override scripts
+		if test -x "$symlink_target" &&
+		    test ! -d "$symlink_target" &&
+		    test "#!" != "$(head -c 2 < "$symlink_target")"
+		then
+			symlink_target=$TEST_DIRECTORY/valgrind.sh
+		fi
+		case "$base" in
+		*.sh|*.perl)
+			symlink_target=$TEST_DIRECTORY/unprocessed-script
+		esac
+		# create the link, or replace it if it is out of date
+		make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
+	}
+
+	# override notmuch executable in TEST_DIRECTORY/..
+	GIT_VALGRIND=$TEST_DIRECTORY/valgrind
+	mkdir -p "$GIT_VALGRIND"/bin
+	make_valgrind_symlink $TEST_DIRECTORY/../notmuch
+	OLDIFS=$IFS
+	IFS=:
+	for path in $PATH
+	do
+		ls "$path"/notmuch 2> /dev/null |
+		while read file
+		do
+			make_valgrind_symlink "$file"
+		done
+	done
+	IFS=$OLDIFS
+	PATH=$GIT_VALGRIND/bin:$PATH
+	GIT_EXEC_PATH=$GIT_VALGRIND/bin
+	export GIT_VALGRIND
+	test -n "$notmuch_path" && MANPATH="$notmuch_path/man:$MANPATH"
+else # normal case
+	if test -n "$notmuch_path"
+		then
+			PATH="$notmuch_path:$PATH"
+			MANPATH="$notmuch_path/man:$MANPATH"
+		fi
+fi
+export PATH MANPATH
+
+# Test repository
+test="tmp.$(basename "$0" .sh)"
+test -n "$root" && test="$root/$test"
+case "$test" in
+/*) TMP_DIRECTORY="$test" ;;
+ *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;;
+esac
+test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
+rm -fr "$test" || {
+	GIT_EXIT_OK=t
+	echo >&5 "FATAL: Cannot prepare test area"
+	exit 1
+}
+
+# A temporary home directory is needed by at least:
+# - emacs/"Sending a message via (fake) SMTP"
+# - emacs/"Reply within emacs"
+# - crypto/emacs_deliver_message
+export HOME="${TMP_DIRECTORY}/home"
+mkdir -p "${HOME}"
+
+MAIL_DIR="${TMP_DIRECTORY}/mail"
+export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
+export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
+
+mkdir -p "${test}"
+mkdir -p "${MAIL_DIR}"
+
+cat <<EOF >"${NOTMUCH_CONFIG}"
+[database]
+path=${MAIL_DIR}
+
+[user]
+name=Notmuch Test Suite
+primary_email=test_suite@notmuchmail.org
+other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
+EOF
diff --git a/test/test-lib.sh b/test/test-lib.sh
index e7638be..6548b29 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -1070,134 +1070,7 @@ test_init_ () {
 }
 
 
-find_notmuch_path ()
-{
-    dir="$1"
-
-    while [ -n "$dir" ]; do
-	bin="$dir/notmuch"
-	if [ -x "$bin" ]; then
-	    echo "$dir"
-	    return
-	fi
-	dir="$(dirname "$dir")"
-	if [ "$dir" = "/" ]; then
-	    break
-	fi
-    done
-}
-
-# Test the binaries we have just built.  The tests are kept in
-# test/ subdirectory and are run in 'trash directory' subdirectory.
-TEST_DIRECTORY=$(pwd)
-notmuch_path=`find_notmuch_path "$TEST_DIRECTORY"`
-if test -n "$valgrind"
-then
-	make_symlink () {
-		test -h "$2" &&
-		test "$1" = "$(readlink "$2")" || {
-			# be super paranoid
-			if mkdir "$2".lock
-			then
-				rm -f "$2" &&
-				ln -s "$1" "$2" &&
-				rm -r "$2".lock
-			else
-				while test -d "$2".lock
-				do
-					say "Waiting for lock on $2."
-					sleep 1
-				done
-			fi
-		}
-	}
-
-	make_valgrind_symlink () {
-		# handle only executables
-		test -x "$1" || return
-
-		base=$(basename "$1")
-		symlink_target=$TEST_DIRECTORY/../$base
-		# do not override scripts
-		if test -x "$symlink_target" &&
-		    test ! -d "$symlink_target" &&
-		    test "#!" != "$(head -c 2 < "$symlink_target")"
-		then
-			symlink_target=$TEST_DIRECTORY/valgrind.sh
-		fi
-		case "$base" in
-		*.sh|*.perl)
-			symlink_target=$TEST_DIRECTORY/unprocessed-script
-		esac
-		# create the link, or replace it if it is out of date
-		make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
-	}
-
-	# override notmuch executable in TEST_DIRECTORY/..
-	GIT_VALGRIND=$TEST_DIRECTORY/valgrind
-	mkdir -p "$GIT_VALGRIND"/bin
-	make_valgrind_symlink $TEST_DIRECTORY/../notmuch
-	OLDIFS=$IFS
-	IFS=:
-	for path in $PATH
-	do
-		ls "$path"/notmuch 2> /dev/null |
-		while read file
-		do
-			make_valgrind_symlink "$file"
-		done
-	done
-	IFS=$OLDIFS
-	PATH=$GIT_VALGRIND/bin:$PATH
-	GIT_EXEC_PATH=$GIT_VALGRIND/bin
-	export GIT_VALGRIND
-	test -n "$notmuch_path" && MANPATH="$notmuch_path/man:$MANPATH"
-else # normal case
-	if test -n "$notmuch_path"
-		then
-			PATH="$notmuch_path:$PATH"
-			MANPATH="$notmuch_path/man:$MANPATH"
-		fi
-fi
-export PATH MANPATH
-
-# Test repository
-test="tmp.$(basename "$0" .sh)"
-test -n "$root" && test="$root/$test"
-case "$test" in
-/*) TMP_DIRECTORY="$test" ;;
- *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;;
-esac
-test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
-rm -fr "$test" || {
-	GIT_EXIT_OK=t
-	echo >&5 "FATAL: Cannot prepare test area"
-	exit 1
-}
-
-# A temporary home directory is needed by at least:
-# - emacs/"Sending a message via (fake) SMTP"
-# - emacs/"Reply within emacs"
-# - crypto/emacs_deliver_message
-export HOME="${TMP_DIRECTORY}/home"
-mkdir -p "${HOME}"
-
-MAIL_DIR="${TMP_DIRECTORY}/mail"
-export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
-export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
-
-mkdir -p "${test}"
-mkdir -p "${MAIL_DIR}"
-
-cat <<EOF >"${NOTMUCH_CONFIG}"
-[database]
-path=${MAIL_DIR}
-
-[user]
-name=Notmuch Test Suite
-primary_email=test_suite@notmuchmail.org
-other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
-EOF
+. test-lib-common.sh
 
 emacs_generate_script
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 5+ messages in thread

* [PATCH 2/2] test: initial performance testing infrastructure.
  2012-11-17 21:37 initial attempt at performance testing suite for notmuch david
  2012-11-17 21:37 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
@ 2012-11-17 21:37 ` david
  2012-11-20  2:23 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
  2 siblings, 0 replies; 5+ messages in thread
From: david @ 2012-11-17 21:37 UTC (permalink / raw)
  To: notmuch; +Cc: David Bremner

From: David Bremner <bremner@debian.org>

This is not near as fancy as as the unit tests, on the theory that
the code should typically be crashing when performance tuning.
Nonetheless, there is plenty of room for improvement.  Several more of
the pieces of the test infrastructure (e.g. the option parsing) could
be factored out into test/test-lib-common.sh
---
 Makefile                                           |    3 +-
 performance-test/.gitignore                        |    2 ++
 performance-test/Makefile                          |    7 ++++
 performance-test/Makefile.local                    |   28 +++++++++++++++
 performance-test/README                            |   26 ++++++++++++++
 performance-test/basic                             |   12 +++++++
 performance-test/download/.gitignore               |    2 ++
 .../download/mail-corpus-0.1.mbox.sha256           |    1 +
 performance-test/notmuch-perf-test                 |   25 ++++++++++++++
 performance-test/perf-test-lib.sh                  |   36 ++++++++++++++++++++
 10 files changed, 141 insertions(+), 1 deletion(-)
 create mode 100644 performance-test/.gitignore
 create mode 100644 performance-test/Makefile
 create mode 100644 performance-test/Makefile.local
 create mode 100644 performance-test/README
 create mode 100755 performance-test/basic
 create mode 100644 performance-test/download/.gitignore
 create mode 100644 performance-test/download/mail-corpus-0.1.mbox.sha256
 create mode 100755 performance-test/notmuch-perf-test
 create mode 100644 performance-test/perf-test-lib.sh

diff --git a/Makefile b/Makefile
index bb9c316..5decbea 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,8 @@
 all:
 
 # List all subdirectories here. Each contains its own Makefile.local
-subdirs = compat completion emacs lib man parse-time-string util test
+subdirs := compat completion emacs lib man parse-time-string
+subdirs := $(subdirs) performance-test util test
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/performance-test/.gitignore b/performance-test/.gitignore
new file mode 100644
index 0000000..e757e8d
--- /dev/null
+++ b/performance-test/.gitignore
@@ -0,0 +1,2 @@
+corpus/*
+tmp.*/
diff --git a/performance-test/Makefile b/performance-test/Makefile
new file mode 100644
index 0000000..de492a7
--- /dev/null
+++ b/performance-test/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C .. all
+
+.DEFAULT:
+	$(MAKE) -C .. $@
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
new file mode 100644
index 0000000..b38bbbd
--- /dev/null
+++ b/performance-test/Makefile.local
@@ -0,0 +1,28 @@
+# -*- makefile -*-
+
+dir := performance-test
+
+PERFTEST_VERSION := 0.1
+
+MBNAME := mail-corpus-$(PERFTEST_VERSION).mbox
+GZFILE := $(CURDIR)/$(dir)/download/$(MBNAME).gz
+MBFILE := $(CURDIR)/$(dir)/download/$(MBNAME)
+MBDIR :=  $(CURDIR)/$(dir)/corpus
+SHA256FILE := $(dir)/download/mail-corpus-$(PERFTEST_VERSION).sha256
+TEST_SCRIPT := $(dir)/notmuch-perf-test
+
+perf-test: setup-perf-test all
+	$(TEST_SCRIPT) $(OPTIONS)
+
+setup-perf-test: $(MBFILE)
+	mb2md -s $(MBFILE) -d $(MBDIR)
+
+$(GZFILE):
+	wget -O $@ http://notmuchmail.org/releases/$(MBNAME).gz
+
+$(MBFILE): $(GZFILE)
+	gunzip -c $(GZFILE) > $(MBFILE)
+	sha256sum -c $(MBFILE).sha256
+
+CLEAN := $(CLEAN) $(dir)/tmp.*
+DISTCLEAN := $(DISTCLEAN) $(GZFILE) $(MBFILE) $(dir)/corpus
diff --git a/performance-test/README b/performance-test/README
new file mode 100644
index 0000000..d22438a
--- /dev/null
+++ b/performance-test/README
@@ -0,0 +1,26 @@
+Pre-requisites
+--------------
+
+In addition to having notmuch, you need:
+
+- mb2md
+- gnu time
+- wget
+- sha256sum
+
+Getting set up to run tests:
+----------------------------
+
+% make setup-perf-test
+
+This requires a net connection to download the test corpus.
+
+Running tests
+-------------
+
+The easiest way to run performance tests is to say "make perf-test", (or
+simply run the notmuch-perf-test script). Either command will run all
+available performance tests.
+
+Alternately, you can run a specific subset of tests by simply invoking
+one of the executable scripts in this directory, (such as ./basic).
diff --git a/performance-test/basic b/performance-test/basic
new file mode 100755
index 0000000..16b7d03
--- /dev/null
+++ b/performance-test/basic
@@ -0,0 +1,12 @@
+
+. ./perf-test-lib.sh
+
+add_email_corpus
+
+time_run 'initial notmuch new' 'notmuch new'
+time_run 'second notmuch new' 'notmuch new'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+
+time_done
diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore
new file mode 100644
index 0000000..b08aedd
--- /dev/null
+++ b/performance-test/download/.gitignore
@@ -0,0 +1,2 @@
+*.mbox.gz
+*.mbox
diff --git a/performance-test/download/mail-corpus-0.1.mbox.sha256 b/performance-test/download/mail-corpus-0.1.mbox.sha256
new file mode 100644
index 0000000..85354c0
--- /dev/null
+++ b/performance-test/download/mail-corpus-0.1.mbox.sha256
@@ -0,0 +1 @@
+ed1163e6e331691d60af8f599866a5d16fa0bbc38793407227fd4b2381494b4b  performance-test/download/mail-corpus-0.1.mbox
diff --git a/performance-test/notmuch-perf-test b/performance-test/notmuch-perf-test
new file mode 100755
index 0000000..1bea345
--- /dev/null
+++ b/performance-test/notmuch-perf-test
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Run tests
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+# Adapted from a Makefile to a shell script by Carl Worth (2010)
+
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+    echo "Error: The notmuch test suite requires a bash version >= 4.0"
+    echo "due to use of associative arrays within the test suite."
+    echo "Please try again with a newer bash (or help us fix the"
+    echo "test suite to be more portable). Thanks."
+    exit 1
+fi
+
+cd $(dirname "$0")
+
+TESTS="
+  basic
+"
+
+for test in $TESTS; do
+    ./$test "$@"
+done
diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh
new file mode 100644
index 0000000..0447852
--- /dev/null
+++ b/performance-test/perf-test-lib.sh
@@ -0,0 +1,36 @@
+
+. ../test/test-lib-common.sh
+
+set -e
+
+if ! test -x ../notmuch
+then
+	echo >&2 'You do not seem to have built notmuch yet.'
+	exit 1
+fi
+
+add_email_corpus ()
+{
+    rm -rf ${MAIL_DIR}
+    cp -a $TEST_DIRECTORY/corpus ${MAIL_DIR}
+}
+
+time_run () {
+    printf "%-25s" "$1"
+    if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+    if ! eval >&3 "/usr/bin/time -f '%e %U %S' $2" ; then
+	test_failure=$(($test_failure + 1))
+    fi
+}
+
+time_done () {
+    if [ "$test_failure" = "0" ]; then
+	rm -rf "$remove_tmp"
+	exit 0
+    else
+	exit 1
+    fi
+}
+
+cd -P "$test" || error "Cannot setup test environment"
+test_failure=0
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 5+ messages in thread

* [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh
  2012-11-17 21:37 initial attempt at performance testing suite for notmuch david
  2012-11-17 21:37 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
  2012-11-17 21:37 ` [PATCH 2/2] test: initial performance testing infrastructure david
@ 2012-11-20  2:23 ` david
  2012-11-20  2:23   ` [PATCH 2/2] test: initial performance testing infrastructure david
  2 siblings, 1 reply; 5+ messages in thread
From: david @ 2012-11-20  2:23 UTC (permalink / raw)
  To: notmuch; +Cc: David Bremner

From: David Bremner <bremner@debian.org>

The idea is to use some of the simpler parts of the test suite
infrastructure to help run performance tests.
---
 test/test-lib-common.sh |  147 +++++++++++++++++++++++++++++++++++++++++++++++
 test/test-lib.sh        |  129 +----------------------------------------
 2 files changed, 148 insertions(+), 128 deletions(-)
 create mode 100644 test/test-lib-common.sh

diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh
new file mode 100644
index 0000000..e1eaa5a
--- /dev/null
+++ b/test/test-lib-common.sh
@@ -0,0 +1,147 @@
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see http://www.gnu.org/licenses/ .
+
+# This file contains common code to be used by both the regular
+# (correctness) tests and the performance tests.
+
+find_notmuch_path ()
+{
+    dir="$1"
+
+    while [ -n "$dir" ]; do
+	bin="$dir/notmuch"
+	if [ -x "$bin" ]; then
+	    echo "$dir"
+	    return
+	fi
+	dir="$(dirname "$dir")"
+	if [ "$dir" = "/" ]; then
+	    break
+	fi
+    done
+}
+
+# Test the binaries we have just built.  The tests are kept in
+# test/ subdirectory and are run in 'trash directory' subdirectory.
+TEST_DIRECTORY=$(pwd)
+notmuch_path=`find_notmuch_path "$TEST_DIRECTORY"`
+if test -n "$valgrind"
+then
+	make_symlink () {
+		test -h "$2" &&
+		test "$1" = "$(readlink "$2")" || {
+			# be super paranoid
+			if mkdir "$2".lock
+			then
+				rm -f "$2" &&
+				ln -s "$1" "$2" &&
+				rm -r "$2".lock
+			else
+				while test -d "$2".lock
+				do
+					say "Waiting for lock on $2."
+					sleep 1
+				done
+			fi
+		}
+	}
+
+	make_valgrind_symlink () {
+		# handle only executables
+		test -x "$1" || return
+
+		base=$(basename "$1")
+		symlink_target=$TEST_DIRECTORY/../$base
+		# do not override scripts
+		if test -x "$symlink_target" &&
+		    test ! -d "$symlink_target" &&
+		    test "#!" != "$(head -c 2 < "$symlink_target")"
+		then
+			symlink_target=$TEST_DIRECTORY/valgrind.sh
+		fi
+		case "$base" in
+		*.sh|*.perl)
+			symlink_target=$TEST_DIRECTORY/unprocessed-script
+		esac
+		# create the link, or replace it if it is out of date
+		make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
+	}
+
+	# override notmuch executable in TEST_DIRECTORY/..
+	GIT_VALGRIND=$TEST_DIRECTORY/valgrind
+	mkdir -p "$GIT_VALGRIND"/bin
+	make_valgrind_symlink $TEST_DIRECTORY/../notmuch
+	OLDIFS=$IFS
+	IFS=:
+	for path in $PATH
+	do
+		ls "$path"/notmuch 2> /dev/null |
+		while read file
+		do
+			make_valgrind_symlink "$file"
+		done
+	done
+	IFS=$OLDIFS
+	PATH=$GIT_VALGRIND/bin:$PATH
+	GIT_EXEC_PATH=$GIT_VALGRIND/bin
+	export GIT_VALGRIND
+	test -n "$notmuch_path" && MANPATH="$notmuch_path/man:$MANPATH"
+else # normal case
+	if test -n "$notmuch_path"
+		then
+			PATH="$notmuch_path:$PATH"
+			MANPATH="$notmuch_path/man:$MANPATH"
+		fi
+fi
+export PATH MANPATH
+
+# Test repository
+test="tmp.$(basename "$0" .sh)"
+test -n "$root" && test="$root/$test"
+case "$test" in
+/*) TMP_DIRECTORY="$test" ;;
+ *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;;
+esac
+test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
+rm -fr "$test" || {
+	GIT_EXIT_OK=t
+	echo >&5 "FATAL: Cannot prepare test area"
+	exit 1
+}
+
+# A temporary home directory is needed by at least:
+# - emacs/"Sending a message via (fake) SMTP"
+# - emacs/"Reply within emacs"
+# - crypto/emacs_deliver_message
+export HOME="${TMP_DIRECTORY}/home"
+mkdir -p "${HOME}"
+
+MAIL_DIR="${TMP_DIRECTORY}/mail"
+export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
+export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
+
+mkdir -p "${test}"
+mkdir -p "${MAIL_DIR}"
+
+cat <<EOF >"${NOTMUCH_CONFIG}"
+[database]
+path=${MAIL_DIR}
+
+[user]
+name=Notmuch Test Suite
+primary_email=test_suite@notmuchmail.org
+other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
+EOF
diff --git a/test/test-lib.sh b/test/test-lib.sh
index e7638be..6548b29 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -1070,134 +1070,7 @@ test_init_ () {
 }
 
 
-find_notmuch_path ()
-{
-    dir="$1"
-
-    while [ -n "$dir" ]; do
-	bin="$dir/notmuch"
-	if [ -x "$bin" ]; then
-	    echo "$dir"
-	    return
-	fi
-	dir="$(dirname "$dir")"
-	if [ "$dir" = "/" ]; then
-	    break
-	fi
-    done
-}
-
-# Test the binaries we have just built.  The tests are kept in
-# test/ subdirectory and are run in 'trash directory' subdirectory.
-TEST_DIRECTORY=$(pwd)
-notmuch_path=`find_notmuch_path "$TEST_DIRECTORY"`
-if test -n "$valgrind"
-then
-	make_symlink () {
-		test -h "$2" &&
-		test "$1" = "$(readlink "$2")" || {
-			# be super paranoid
-			if mkdir "$2".lock
-			then
-				rm -f "$2" &&
-				ln -s "$1" "$2" &&
-				rm -r "$2".lock
-			else
-				while test -d "$2".lock
-				do
-					say "Waiting for lock on $2."
-					sleep 1
-				done
-			fi
-		}
-	}
-
-	make_valgrind_symlink () {
-		# handle only executables
-		test -x "$1" || return
-
-		base=$(basename "$1")
-		symlink_target=$TEST_DIRECTORY/../$base
-		# do not override scripts
-		if test -x "$symlink_target" &&
-		    test ! -d "$symlink_target" &&
-		    test "#!" != "$(head -c 2 < "$symlink_target")"
-		then
-			symlink_target=$TEST_DIRECTORY/valgrind.sh
-		fi
-		case "$base" in
-		*.sh|*.perl)
-			symlink_target=$TEST_DIRECTORY/unprocessed-script
-		esac
-		# create the link, or replace it if it is out of date
-		make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
-	}
-
-	# override notmuch executable in TEST_DIRECTORY/..
-	GIT_VALGRIND=$TEST_DIRECTORY/valgrind
-	mkdir -p "$GIT_VALGRIND"/bin
-	make_valgrind_symlink $TEST_DIRECTORY/../notmuch
-	OLDIFS=$IFS
-	IFS=:
-	for path in $PATH
-	do
-		ls "$path"/notmuch 2> /dev/null |
-		while read file
-		do
-			make_valgrind_symlink "$file"
-		done
-	done
-	IFS=$OLDIFS
-	PATH=$GIT_VALGRIND/bin:$PATH
-	GIT_EXEC_PATH=$GIT_VALGRIND/bin
-	export GIT_VALGRIND
-	test -n "$notmuch_path" && MANPATH="$notmuch_path/man:$MANPATH"
-else # normal case
-	if test -n "$notmuch_path"
-		then
-			PATH="$notmuch_path:$PATH"
-			MANPATH="$notmuch_path/man:$MANPATH"
-		fi
-fi
-export PATH MANPATH
-
-# Test repository
-test="tmp.$(basename "$0" .sh)"
-test -n "$root" && test="$root/$test"
-case "$test" in
-/*) TMP_DIRECTORY="$test" ;;
- *) TMP_DIRECTORY="$TEST_DIRECTORY/$test" ;;
-esac
-test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
-rm -fr "$test" || {
-	GIT_EXIT_OK=t
-	echo >&5 "FATAL: Cannot prepare test area"
-	exit 1
-}
-
-# A temporary home directory is needed by at least:
-# - emacs/"Sending a message via (fake) SMTP"
-# - emacs/"Reply within emacs"
-# - crypto/emacs_deliver_message
-export HOME="${TMP_DIRECTORY}/home"
-mkdir -p "${HOME}"
-
-MAIL_DIR="${TMP_DIRECTORY}/mail"
-export GNUPGHOME="${TMP_DIRECTORY}/gnupg"
-export NOTMUCH_CONFIG="${TMP_DIRECTORY}/notmuch-config"
-
-mkdir -p "${test}"
-mkdir -p "${MAIL_DIR}"
-
-cat <<EOF >"${NOTMUCH_CONFIG}"
-[database]
-path=${MAIL_DIR}
-
-[user]
-name=Notmuch Test Suite
-primary_email=test_suite@notmuchmail.org
-other_email=test_suite_other@notmuchmail.org;test_suite@otherdomain.org
-EOF
+. test-lib-common.sh
 
 emacs_generate_script
 
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 5+ messages in thread

* [PATCH 2/2] test: initial performance testing infrastructure
  2012-11-20  2:23 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
@ 2012-11-20  2:23   ` david
  0 siblings, 0 replies; 5+ messages in thread
From: david @ 2012-11-20  2:23 UTC (permalink / raw)
  To: notmuch; +Cc: David Bremner

From: David Bremner <bremner@debian.org>

This is not near as fancy as as the unit tests, on the theory that
the code should typically be crashing when performance tuning.
Nonetheless, there is plenty of room for improvement.  Several more of
the pieces of the test infrastructure (e.g. the option parsing) could
be factored out into test/test-lib-common.sh
---
 Makefile                                           |    3 +-
 performance-test/.gitignore                        |    1 +
 performance-test/Makefile                          |    7 +++
 performance-test/Makefile.local                    |   24 +++++++++
 performance-test/README                            |   36 +++++++++++++
 performance-test/basic                             |   15 ++++++
 performance-test/download/.gitignore               |    1 +
 .../download/notmuch-email-corpus-0.1.tar.gz.asc   |    9 ++++
 performance-test/notmuch-perf-test                 |   25 +++++++++
 performance-test/perf-test-lib.sh                  |   57 ++++++++++++++++++++
 10 files changed, 177 insertions(+), 1 deletion(-)
 create mode 100644 performance-test/.gitignore
 create mode 100644 performance-test/Makefile
 create mode 100644 performance-test/Makefile.local
 create mode 100644 performance-test/README
 create mode 100755 performance-test/basic
 create mode 100644 performance-test/download/.gitignore
 create mode 100644 performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc
 create mode 100755 performance-test/notmuch-perf-test
 create mode 100644 performance-test/perf-test-lib.sh

diff --git a/Makefile b/Makefile
index bb9c316..5decbea 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,8 @@
 all:
 
 # List all subdirectories here. Each contains its own Makefile.local
-subdirs = compat completion emacs lib man parse-time-string util test
+subdirs := compat completion emacs lib man parse-time-string
+subdirs := $(subdirs) performance-test util test
 
 # We make all targets depend on the Makefiles themselves.
 global_deps = Makefile Makefile.config Makefile.local \
diff --git a/performance-test/.gitignore b/performance-test/.gitignore
new file mode 100644
index 0000000..53f2697
--- /dev/null
+++ b/performance-test/.gitignore
@@ -0,0 +1 @@
+tmp.*/
diff --git a/performance-test/Makefile b/performance-test/Makefile
new file mode 100644
index 0000000..de492a7
--- /dev/null
+++ b/performance-test/Makefile
@@ -0,0 +1,7 @@
+# See Makefile.local for the list of files to be compiled in this
+# directory.
+all:
+	$(MAKE) -C .. all
+
+.DEFAULT:
+	$(MAKE) -C .. $@
diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local
new file mode 100644
index 0000000..d20457e
--- /dev/null
+++ b/performance-test/Makefile.local
@@ -0,0 +1,24 @@
+# -*- makefile -*-
+
+dir := performance-test
+
+PERFTEST_VERSION := 0.1
+
+TGZFILE := $(dir)/download/notmuch-email-corpus-$(PERFTEST_VERSION).tar.gz
+SIGFILE := $(TGZFILE).asc
+TEST_SCRIPT := $(dir)/notmuch-perf-test
+
+perf-test: setup-perf-test all
+	$(TEST_SCRIPT) $(OPTIONS)
+
+setup-perf-test: $(TGZFILE)
+	gpg --verify $(SIGFILE)
+
+$(TGZFILE):
+	@echo
+	@echo Please download ${TGZFILE}
+	@echo See http://notmuchmail.org/corpus for download locations
+	@echo
+	@false
+
+CLEAN := $(CLEAN) $(dir)/tmp.*
diff --git a/performance-test/README b/performance-test/README
new file mode 100644
index 0000000..1f98337
--- /dev/null
+++ b/performance-test/README
@@ -0,0 +1,36 @@
+Pre-requisites
+--------------
+
+In addition to having notmuch, you need:
+
+- gpg
+- gnu tar
+- gnu time
+
+Getting set up to run tests:
+----------------------------
+
+First, you need to get the corpus.
+
+It should work to run
+
+   % cd download
+   % wget http://notmuchmail.org/releases/notmuch-email-corpus-${V}.tar.gz
+
+Where $V is the current version.
+
+In case that fails or is too slow, check
+
+   http://notmuchmail.org/corpus
+
+for a list of mirrors.
+
+Running tests
+-------------
+
+The easiest way to run performance tests is to say "make perf-test", (or
+simply run the notmuch-perf-test script). Either command will run all
+available performance tests.
+
+Alternately, you can run a specific subset of tests by simply invoking
+one of the executable scripts in this directory, (such as ./basic).
diff --git a/performance-test/basic b/performance-test/basic
new file mode 100755
index 0000000..9d015ee
--- /dev/null
+++ b/performance-test/basic
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+. ./perf-test-lib.sh
+
+add_email_corpus
+
+print_header
+
+time_run 'initial notmuch new' 'notmuch new'
+time_run 'second notmuch new' 'notmuch new'
+time_run 'dump *' 'notmuch dump > tags.out'
+time_run 'restore *' 'notmuch restore < tags.out'
+time_run 'tag * +new_tag' "notmuch tag +new_tag '*'"
+
+time_done
diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore
new file mode 100644
index 0000000..335ec95
--- /dev/null
+++ b/performance-test/download/.gitignore
@@ -0,0 +1 @@
+*.tar.gz
diff --git a/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc b/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc
new file mode 100644
index 0000000..1041523
--- /dev/null
+++ b/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc
@@ -0,0 +1,9 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iJwEAAECAAYFAlCqaPMACgkQTiiN/0Um85mDdQP/c+AErVD2ix+1Jf159EigEpND
+Kzmak2c0Ae2bsfkjZa07GIc9T147lloSg+TKO3PMXTaXHCyYYolgNjZsD8SnP3Hp
+nF2zgNrRlVpaMysqJX005OMbb3cVUrKHQpOCZ764dFyVPCi7ag42HRy/XrODUD3w
+HJsx8ejTFbxfYi9c2+g=
+=4ZIO
+-----END PGP SIGNATURE-----
diff --git a/performance-test/notmuch-perf-test b/performance-test/notmuch-perf-test
new file mode 100755
index 0000000..1bea345
--- /dev/null
+++ b/performance-test/notmuch-perf-test
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Run tests
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+# Adapted from a Makefile to a shell script by Carl Worth (2010)
+
+if [ ${BASH_VERSINFO[0]} -lt 4 ]; then
+    echo "Error: The notmuch test suite requires a bash version >= 4.0"
+    echo "due to use of associative arrays within the test suite."
+    echo "Please try again with a newer bash (or help us fix the"
+    echo "test suite to be more portable). Thanks."
+    exit 1
+fi
+
+cd $(dirname "$0")
+
+TESTS="
+  basic
+"
+
+for test in $TESTS; do
+    ./$test "$@"
+done
diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh
new file mode 100644
index 0000000..8e6004f
--- /dev/null
+++ b/performance-test/perf-test-lib.sh
@@ -0,0 +1,57 @@
+
+. ../test/test-lib-common.sh
+
+set -e
+
+if ! test -x ../notmuch
+then
+	echo >&2 'You do not seem to have built notmuch yet.'
+	exit 1
+fi
+
+add_email_corpus ()
+{
+    rm -rf ${MAIL_DIR}
+
+    arg=""
+    case "$1" in
+	--small)
+	    arg="/enron/bailey-s"
+	    ;;
+	--medium)
+	    arg="/notmuch-archive"
+	    ;;
+    esac
+
+    printf "Unpacking corpus\n"
+    tar --checkpoint=.5000 --extract --strip-components=1 \
+	--directory ${TMP_DIRECTORY} \
+	--file ../download/notmuch-email-corpus-0.1.tar.gz \
+	notmuch-email-corpus/mail"$arg"
+
+    printf "\n"
+}
+
+print_header () {
+    printf "                      Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn\tOut\n"
+}
+
+time_run () {
+    printf "%-22s" "$1"
+    if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+    if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I\t%O' $2" ; then
+	test_failure=$(($test_failure + 1))
+    fi
+}
+
+time_done () {
+    if [ "$test_failure" = "0" ]; then
+	rm -rf "$remove_tmp"
+	exit 0
+    else
+	exit 1
+    fi
+}
+
+cd -P "$test" || error "Cannot setup test environment"
+test_failure=0
-- 
1.7.10.4

^ permalink raw reply related	[flat|nested] 5+ messages in thread

end of thread, other threads:[~2012-11-20  2:23 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2012-11-17 21:37 initial attempt at performance testing suite for notmuch david
2012-11-17 21:37 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
2012-11-17 21:37 ` [PATCH 2/2] test: initial performance testing infrastructure david
2012-11-20  2:23 ` [PATCH 1/2] test: factor out part of test-lib.sh into test-lib-common.sh david
2012-11-20  2:23   ` [PATCH 2/2] test: initial performance testing infrastructure david

Code repositories for project(s) associated with this public inbox

	https://yhetil.org/notmuch.git/

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).