* [PATCH 1/3] WIP/test: support testing notmuch as installed
2021-10-25 14:57 WIP: Support testing notmuch as installed David Bremner
@ 2021-10-25 14:57 ` David Bremner
2021-10-25 14:57 ` [PATCH 2/3] debian: add autopkgtests David Bremner
2021-10-25 14:57 ` [PATCH 3/3] test: mark some tests as broken when run as root David Bremner
2 siblings, 0 replies; 4+ messages in thread
From: David Bremner @ 2021-10-25 14:57 UTC (permalink / raw)
To: notmuch
We put some effort into testing the built copy rather than some
installed copy. On the other hand for people like packagers, testing
the installed copy is also of interest.
When NOTMUCH_TEST_INSTALLED is set to a nonempty value, tests do not
require a built notmuch tree or running configure.
One questionable aspect is the file "defaults.sh" which is used to
fake a certain set of minimal responses to configure.
---
test/T000-basic.sh | 1 +
test/T240-dump-restore.sh | 2 ++
test/T310-emacs.sh | 2 ++
test/T360-symbol-hiding.sh | 4 ++++
test/T390-python.sh | 4 ++++
test/T391-python-cffi.sh | 5 ++++-
test/T395-ruby.sh | 8 ++++++--
test/T410-argument-parsing.sh | 4 ++++
test/T480-hex-escaping.sh | 4 ++++
test/T490-parse-time-string.sh | 4 ++++
test/T550-db-features.sh | 4 ++++
| 4 ++++
test/T710-message-id.sh | 4 ++++
test/defaults.sh | 3 +++
test/export-dirs.sh | 2 +-
test/notmuch-test | 12 ++++++++++--
test/test-lib-common.sh | 16 +++++++++++++---
test/test-lib-emacs.sh | 14 +++++++++++---
test/test-lib.sh | 27 +++++++++++++++++++++------
19 files changed, 106 insertions(+), 18 deletions(-)
create mode 100644 test/defaults.sh
diff --git a/test/T000-basic.sh b/test/T000-basic.sh
index a2f4d93f..642f918d 100755
--- a/test/T000-basic.sh
+++ b/test/T000-basic.sh
@@ -66,6 +66,7 @@ test_begin_subtest 'NOTMUCH_CONFIG is set and points to an existing file'
test_expect_success 'test -f "${NOTMUCH_CONFIG}"'
test_begin_subtest 'PATH is set to build directory'
+test_subtest_broken_for_installed
test_expect_equal \
"$(dirname ${TEST_DIRECTORY})" \
"$(echo $PATH|cut -f1 -d: | sed -e 's,/test/valgrind/bin$,,')"
diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh
index a86f0fb7..8848e06c 100755
--- a/test/T240-dump-restore.sh
+++ b/test/T240-dump-restore.sh
@@ -139,6 +139,7 @@ notmuch dump --output=dump-outfile-dash-inbox.actual -- from:cworth
test_expect_equal_file dump-cworth.expected dump-outfile-dash-inbox.actual
test_begin_subtest "Check for a safe set of message-ids"
+test_subtest_broken_for_installed
notmuch search --output=messages from:cworth | sed s/^id:// > EXPECTED
notmuch search --output=messages from:cworth | sed s/^id:// |\
$TEST_DIRECTORY/hex-xcode --direction=encode > OUTPUT
@@ -246,6 +247,7 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count
test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
test_begin_subtest 'format=batch-tag, checking encoded output'
+test_subtest_broken_for_installed
NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\
awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count
NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth > OUTPUT.$test_count
diff --git a/test/T310-emacs.sh b/test/T310-emacs.sh
index 1b6660f0..e3b6de2a 100755
--- a/test/T310-emacs.sh
+++ b/test/T310-emacs.sh
@@ -327,6 +327,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "Verify that sent messages are saved/searchable (via FCC)"
+test_subtest_broken_for_installed
notmuch new > /dev/null
output=$(notmuch search 'subject:"testing message sent via SMTP"' | notmuch_search_sanitize)
test_expect_equal "$output" "thread:XXX 2000-01-01 [1/1] Notmuch Test Suite; Testing message sent via SMTP (inbox)"
@@ -419,6 +420,7 @@ EOF
test_expect_equal_file EXPECTED OUTPUT
test_begin_subtest "Reply within emacs"
+test_subtest_broken_for_installed
test_emacs '(let ((message-hidden-headers ''()))
(notmuch-search "subject:\"testing message sent via SMTP\"")
(notmuch-test-wait)
diff --git a/test/T360-symbol-hiding.sh b/test/T360-symbol-hiding.sh
index 642457bf..5bf8da14 100755
--- a/test/T360-symbol-hiding.sh
+++ b/test/T360-symbol-hiding.sh
@@ -11,6 +11,10 @@ test_description='exception symbol hiding'
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
test_begin_subtest 'running test' run_test
mkdir -p ${PWD}/fakedb/.notmuch
$TEST_DIRECTORY/symbol-test ${PWD}/fakedb ${PWD}/nonexistent 2>&1 \
diff --git a/test/T390-python.sh b/test/T390-python.sh
index 9f71ce3c..710d2553 100755
--- a/test/T390-python.sh
+++ b/test/T390-python.sh
@@ -4,6 +4,10 @@ test_description="python bindings"
test_require_external_prereq ${NOTMUCH_PYTHON}
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
add_email_corpus
add_gnupg_home
diff --git a/test/T391-python-cffi.sh b/test/T391-python-cffi.sh
index d54bad27..40916e75 100755
--- a/test/T391-python-cffi.sh
+++ b/test/T391-python-cffi.sh
@@ -2,10 +2,13 @@
test_description="python bindings (pytest)"
. $(dirname "$0")/test-lib.sh || exit 1
-if [ $NOTMUCH_HAVE_PYTHON3_CFFI -eq 0 -o $NOTMUCH_HAVE_PYTHON3_PYTEST -eq 0 ]; then
+if [[ -n "${NOTMUCH_TEST_INSTALLED}" ]]; then
test_done
fi
+if [ $NOTMUCH_HAVE_PYTHON3_CFFI -eq 0 -o $NOTMUCH_HAVE_PYTHON3_PYTEST -eq 0 ]; then
+ test_done
+fi
test_begin_subtest "python cffi tests"
pytest_dir=$NOTMUCH_BUILDDIR/bindings/python-cffi/build/stage
diff --git a/test/T395-ruby.sh b/test/T395-ruby.sh
index e828efed..6a5cf615 100755
--- a/test/T395-ruby.sh
+++ b/test/T395-ruby.sh
@@ -2,7 +2,7 @@
test_description="ruby bindings"
. $(dirname "$0")/test-lib.sh || exit 1
-if [ "${NOTMUCH_HAVE_RUBY_DEV}" = "0" ]; then
+if [[ -z "${NOTMUCH_TEST_INSTALLED}" && "${NOTMUCH_HAVE_RUBY_DEV}" = "0" ]]; then
test_subtest_missing_external_prereq_["ruby development files"]=t
fi
@@ -15,7 +15,11 @@ test_ruby() {
db = Notmuch::Database.new('$MAIL_DIR')
EOF
cat
- ) | $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"> OUTPUT
+ ) | if [[ -n "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ ruby
+ else
+ $NOTMUCH_RUBY -I "$NOTMUCH_BUILDDIR/bindings/ruby"
+ fi> OUTPUT
test_expect_equal_file EXPECTED OUTPUT
}
diff --git a/test/T410-argument-parsing.sh b/test/T410-argument-parsing.sh
index d9aa8e2d..5af12916 100755
--- a/test/T410-argument-parsing.sh
+++ b/test/T410-argument-parsing.sh
@@ -2,6 +2,10 @@
test_description="argument parsing"
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
test_begin_subtest "sanity check"
$TEST_DIRECTORY/arg-test pos1 --keyword=one --boolean --string=foo pos2 --int=7 --flag=one --flag=three > OUTPUT
cat <<EOF > EXPECTED
diff --git a/test/T480-hex-escaping.sh b/test/T480-hex-escaping.sh
index b2eb80b9..744d5042 100755
--- a/test/T480-hex-escaping.sh
+++ b/test/T480-hex-escaping.sh
@@ -2,6 +2,10 @@
test_description="hex encoding and decoding"
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
test_begin_subtest "round trip"
find $NOTMUCH_SRCDIR/test/corpora/default -type f -print | sort | xargs cat > EXPECTED
$TEST_DIRECTORY/hex-xcode --direction=encode < EXPECTED | $TEST_DIRECTORY/hex-xcode --direction=decode > OUTPUT
diff --git a/test/T490-parse-time-string.sh b/test/T490-parse-time-string.sh
index f89755ed..fa7d547c 100755
--- a/test/T490-parse-time-string.sh
+++ b/test/T490-parse-time-string.sh
@@ -2,6 +2,10 @@
test_description="date/time parser module"
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
# Sanity/smoke tests for the date/time parser independent of notmuch
_date () {
diff --git a/test/T550-db-features.sh b/test/T550-db-features.sh
index 9d5a9e70..c3ece78b 100755
--- a/test/T550-db-features.sh
+++ b/test/T550-db-features.sh
@@ -3,6 +3,10 @@ test_description="database version and feature compatibility"
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
test_begin_subtest "future database versions abort open"
${TEST_DIRECTORY}/make-db-version ${MAIL_DIR} 9999 ""
output=$(notmuch search x 2>&1 | sed 's/\(database at\) .*/\1 FILENAME/')
--git a/test/T592-thread-breakage.sh b/test/T592-thread-breakage.sh
index 92a70e3e..fd9e628b 100755
--- a/test/T592-thread-breakage.sh
+++ b/test/T592-thread-breakage.sh
@@ -21,6 +21,10 @@ test_description='thread breakage during reindexing'
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n ${NOTMUCH_TEST_INSTALLED} ]]; then
+ test_done
+fi
+
message_a () {
mkdir -p ${MAIL_DIR}/cur
cat > ${MAIL_DIR}/cur/a <<EOF
diff --git a/test/T710-message-id.sh b/test/T710-message-id.sh
index 5129d84c..340ec426 100755
--- a/test/T710-message-id.sh
+++ b/test/T710-message-id.sh
@@ -3,6 +3,10 @@ test_description="message id parsing"
. $(dirname "$0")/test-lib.sh || exit 1
+if [[ -n "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ test_done
+fi
+
test_begin_subtest "good message ids"
${TEST_DIRECTORY}/message-id-parse <<EOF >OUTPUT
<018b1a8f2d1df62e804ce88b65401304832dfbbf.1346614915.git.jani@nikula.org>
diff --git a/test/defaults.sh b/test/defaults.sh
new file mode 100644
index 00000000..565faa2b
--- /dev/null
+++ b/test/defaults.sh
@@ -0,0 +1,3 @@
+NOTMUCH_PYTHON=${NOTMUCH_PYTHON-python3}
+NOTMUCH_HAVE_MAN=${NOTMUCH_HAVE_MAN-1}
+NOTMUCH_HAVE_SFSEXP=1
diff --git a/test/export-dirs.sh b/test/export-dirs.sh
index 0a048e1f..3109da88 100644
--- a/test/export-dirs.sh
+++ b/test/export-dirs.sh
@@ -24,7 +24,7 @@ find_builddir () {
if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then
export NOTMUCH_BUILDDIR="$(find_builddir "$(pwd)")"
- if [[ -z "${NOTMUCH_BUILDDIR}" ]]; then
+ if [[ -z "${NOTMUCH_BUILDDIR}" && -z "${NOTMUCH_TEST_INSTALLED}" ]]; then
echo "Run tests in a subdir of built notmuch tree." >&2
exit 1
fi
diff --git a/test/notmuch-test b/test/notmuch-test
index 14747bdb..5d27e4d1 100755
--- a/test/notmuch-test
+++ b/test/notmuch-test
@@ -20,6 +20,14 @@ fi
set -eu
+# Where to run the tests
+# XXX FIXME this code is duplicated with test-lib.sh
+if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then
+ TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
+else
+ TEST_DIRECTORY=$NOTMUCH_SRCDIR/test
+fi
+
TESTS=
for test in ${NOTMUCH_TESTS-}; do
TESTS="$TESTS $NOTMUCH_SRCDIR/test/$test"
@@ -80,7 +88,7 @@ for file in $TESTS
do
file=${file##*/} # drop leading path components
file=${file%.sh} # drop trailing '.sh'
- RESULT_FILES="$RESULT_FILES $NOTMUCH_BUILDDIR/test/test-results/$file"
+ RESULT_FILES="$RESULT_FILES $TEST_DIRECTORY/test-results/$file"
done
echo
@@ -94,6 +102,6 @@ if [ -n "$META_FAILURE" ]; then
fi
# Clean up
-rm -rf $NOTMUCH_BUILDDIR/test/test-results
+rm -rf $TEST_DIRECTORY/test-results
exit $ev
diff --git a/test/test-lib-common.sh b/test/test-lib-common.sh
index ebbf4cdf..14791bd3 100644
--- a/test/test-lib-common.sh
+++ b/test/test-lib-common.sh
@@ -24,7 +24,7 @@
#
type die >/dev/null 2>&1 || die () { echo "$@" >&2; exit 1; }
-if [[ -z "$NOTMUCH_SRCDIR" ]] || [[ -z "$NOTMUCH_BUILDDIR" ]]; then
+if [[ -z "$NOTMUCH_SRCDIR" ]] || [[ -z "${NOTMUCH_TEST_INSTALLED}" && -z "$NOTMUCH_BUILDDIR" ]]; then
echo "internal: srcdir or builddir not set" >&2
exit 1
fi
@@ -47,7 +47,12 @@ LD_LIBRARY_PATH=${TEST_DIRECTORY%/*}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
export LD_LIBRARY_PATH
# configure output
-. "$NOTMUCH_BUILDDIR/sh.config" || exit 1
+if [[ -z "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ . "$NOTMUCH_BUILDDIR/sh.config" || exit 1
+fi
+
+# fall back definitions for running without configuration
+. "${TEST_DIRECTORY}/defaults.sh"
# load OS specifics
if [[ -e "$NOTMUCH_SRCDIR/test/test-lib-$PLATFORM.sh" ]]; then
@@ -301,7 +306,12 @@ export PATH MANPATH
# Test repository
test="tmp.$(basename "$0" .sh)"
-TMP_DIRECTORY="$TEST_DIRECTORY/$test"
+if [[ -z "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ TMP_DIRECTORY="$TEST_DIRECTORY/$test"
+else
+ TMP_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}/notmuch-$test.XXXXXX")
+fi
+
test ! -z "$debug" || remove_tmp=$TMP_DIRECTORY
rm -rf "$TMP_DIRECTORY" || {
GIT_EXIT_OK=t
diff --git a/test/test-lib-emacs.sh b/test/test-lib-emacs.sh
index dde32177..88ac4d2b 100644
--- a/test/test-lib-emacs.sh
+++ b/test/test-lib-emacs.sh
@@ -30,6 +30,7 @@ test_require_emacs () {
# to the message and encrypting/signing.
emacs_deliver_message () {
local subject body smtp_dummy_pid smtp_dummy_port
+ test_subtest_broken_for_installed
subject="$1"
body="$2"
shift 2
@@ -143,6 +144,13 @@ emacs_generate_script () {
# Construct a little test script here for the benefit of the user,
# (who can easily run "run_emacs" to get the same emacs environment
# for investigating any failures).
+ if [ -z "${NOTMUCH_TEST_INSTALLED}" ]; then
+ find_notmuch_el='--directory "$NOTMUCH_BUILDDIR/emacs"'
+ else
+ ### XXX FIXME: this should really use the installed emacs lisp files
+ find_notmuch_el='--directory "$NOTMUCH_SRCDIR/emacs"'
+ fi
+
cat <<EOF >"$TMP_DIRECTORY/run_emacs"
#!/bin/sh
export PATH=$PATH
@@ -157,12 +165,12 @@ export NOTMUCH_CONFIG=$NOTMUCH_CONFIG
#
# --load Force loading of notmuch.el and test-lib.el
-exec ${TEST_EMACS} --quick \
- --directory "$NOTMUCH_BUILDDIR/emacs" --load notmuch.el \
+exec ${TEST_EMACS} ${find_notmuch_el} --quick \
+ ${EXTRA_DIR} --load notmuch.el \
--directory "$NOTMUCH_SRCDIR/test" --load test-lib.el \
"\$@"
EOF
- chmod a+x "$TMP_DIRECTORY/run_emacs"
+ chmod a+x "$TMP_DIRECTORY/run_emacs"
}
test_emacs () {
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 32f710a5..6713275f 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -29,8 +29,8 @@ shopt -u xpg_echo
# Ensure NOTMUCH_SRCDIR and NOTMUCH_BUILDDIR are set.
. $(dirname "$0")/export-dirs.sh || exit 1
-# It appears that people try to run tests without building...
-if [[ ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then
+# We need either a built tree, or a promise of an installed notmuch
+if [[ -z "${NOTMUCH_TEST_INSTALLED}" && ! -x "$NOTMUCH_BUILDDIR/notmuch" ]]; then
echo >&2 'You do not seem to have built notmuch yet.'
exit 1
fi
@@ -773,6 +773,12 @@ test_subtest_known_broken () {
test_subtest_known_broken_=t
}
+test_subtest_broken_for_installed () {
+ if [[ -n "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ test_subtest_known_broken_=t
+ fi
+}
+
test_expect_success () {
exec 1>&6 2>&7 # Restore stdout and stderr
if [ -z "$inside_subtest" ]; then
@@ -935,11 +941,16 @@ make_shim () {
}
notmuch_with_shim () {
- local base_name shim_file
- base_name="$1"
+ local base_name shim_file notmuch_cmd
+ if [[ -n "${NOTMUCH_TEST_INSTALLED}" ]]; then
+ notmuch_cmd="notmuch"
+ else
+ notmuch_cmd="notmuch-shared"
+ fi
+ base_name=$1
shift
shim_file="${base_name}.so"
- LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} notmuch-shared "$@"
+ LD_PRELOAD=${LD_PRELOAD:+:$LD_PRELOAD}:./${shim_file} $notmuch_cmd "$@"
}
# Creates a script that counts how much time it is executed and calls
@@ -991,7 +1002,11 @@ test_init_ () {
# Where to run the tests
-TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
+if [[ -n "${NOTMUCH_BUILDDIR}" ]]; then
+ TEST_DIRECTORY=$NOTMUCH_BUILDDIR/test
+else
+ TEST_DIRECTORY=$NOTMUCH_SRCDIR/test
+fi
. "$NOTMUCH_SRCDIR/test/test-lib-common.sh" || exit 1
--
2.33.0
^ permalink raw reply related [flat|nested] 4+ messages in thread