[PATCH] Add the beginnings of a test suite.

[PATCH] Add the beginnings of a test suite.

[Jeffrey C. Ollie]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 4428 bytes --]

This is the beginning of a test suite.  It uses the Check[1] unit
testing framework to handle the testing.  There's one basic test of
the SHA1 routines, obviously many more will need to be added.

Run "make check" from the toplevel directory to build and run the
checks.

[1] http://check.sourceforge.net/

Signed-off-by: Jeffrey C. Ollie <jeff@ocjtech.us>
---
 .gitignore            |    1 +
 Makefile              |    1 +
 Makefile.local        |    3 ++
 checks/Makefile.local |   12 +++++++++
 checks/check.c        |   61 +++++++++++++++++++++++++++++++++++++++++++++++++
 configure             |    8 ++++++
 6 files changed, 86 insertions(+), 0 deletions(-)
 create mode 100644 checks/Makefile.local
 create mode 100644 checks/check.c

diff --git a/.gitignore b/.gitignore
index 8794354..bd542d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ notmuch.1.gz
 *~
 .*.swp
 *.elc
+checks/check
diff --git a/Makefile b/Makefile
index 2cd1b1b..1a4c8db 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,7 @@ override LDFLAGS += \
 # Include our local Makefile.local first so that its first target is default
 include Makefile.local
 include lib/Makefile.local
+include checks/Makefile.local
 
 # And get user settings from the output of configure
 include Makefile.config
diff --git a/Makefile.local b/Makefile.local
index 1744747..f6ffd00 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -47,5 +47,8 @@ install-emacs: install emacs
 	install -m0644 notmuch.el $(DESTDIR)$(emacs_lispdir)
 	install -m0644 notmuch.elc $(DESTDIR)$(emacs_lispdir)
 
+check: checks/check
+	checks/check
+
 SRCS  := $(SRCS) $(notmuch_client_srcs)
 CLEAN := $(CLEAN) notmuch $(notmuch_client_modules) notmuch.elc notmuch.1.gz
diff --git a/checks/Makefile.local b/checks/Makefile.local
new file mode 100644
index 0000000..ec34d17
--- /dev/null
+++ b/checks/Makefile.local
@@ -0,0 +1,12 @@
+dir=checks
+extra_cflags += -I../lib
+
+check_c_srcs =		\
+	$(dir)/check.c
+
+check_modules = $(check_c_srcs:.c=.o)
+$(dir)/check: $(check_modules)
+	$(call quiet,CXX) $^ $(LDFLAGS) -o $@ lib/notmuch.a -lcheck
+
+SRCS  := $(SRCS) $(check_c_srcs)
+CLEAN := $(CLEAN) $(check_modules) $(dir)/check
diff --git a/checks/check.c b/checks/check.c
new file mode 100644
index 0000000..8886f1f
--- /dev/null
+++ b/checks/check.c
@@ -0,0 +1,61 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+#include <string.h>
+
+#include "notmuch-private.h"
+
+START_TEST (test_sha1_simple)
+{
+   char *result;
+
+   result = notmuch_sha1_of_string("abcdefghi");
+   fail_if(strcmp("e1435dfb334ec81f2bdd5b0aa45969586e7681c0", result) != 0,
+	   "SHA1 results do not match");
+}
+END_TEST
+
+Suite *
+notmuch_suite (void)
+{
+   Suite *s = suite_create ("NotMuch");
+   
+   /* SHA1 test cases */
+   TCase *tc_sha1 = tcase_create ("SHA1");
+   tcase_add_test (tc_sha1, test_sha1_simple);
+   suite_add_tcase (s, tc_sha1);
+   return s;
+}
+
+int
+main (void)
+{
+   int number_failed;
+
+   Suite *s = notmuch_suite ();
+   SRunner *sr = srunner_create (s);
+
+   srunner_run_all (sr, CK_VERBOSE);
+   number_failed = srunner_ntests_failed (sr);
+   srunner_free (sr);
+
+   return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/configure b/configure
index e55f067..9e76f09 100755
--- a/configure
+++ b/configure
@@ -61,6 +61,14 @@ else
     have_valgrind=
 fi
 
+if pkg-config --exists check > /dev/null 2>&1; then
+    echo "Checking for the Check unit test framework... Yes."
+    have_check=1
+else
+    echo "Checking for the Check unit test framework... No."
+    have_check=
+fi
+
 if [ $errors -gt 0 ]; then
     cat <<EOF
 
-- 
1.6.5.2

[PATCH] Add the beginnings of a test suite.

[Jeffrey C. Ollie]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 5130 bytes --]

This is the beginning of a test suite.  It uses the Check[1] unit
testing framework to handle the testing.  There's are basic tests of
the SHA1 routines, obviously many more will need to be added.

Run "make check" from the toplevel directory to build and run the
checks.

[1] http://check.sourceforge.net/

Signed-off-by: Jeffrey C. Ollie <jeff@ocjtech.us>
---
 .gitignore            |    1 +
 Makefile              |    1 +
 Makefile.local        |    3 ++
 checks/Makefile.local |   12 ++++++
 checks/check.c        |   90 +++++++++++++++++++++++++++++++++++++++++++++++++
 configure             |    8 ++++
 6 files changed, 115 insertions(+), 0 deletions(-)
 create mode 100644 checks/Makefile.local
 create mode 100644 checks/check.c

diff --git a/.gitignore b/.gitignore
index 8794354..bd542d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ notmuch.1.gz
 *~
 .*.swp
 *.elc
+checks/check
diff --git a/Makefile b/Makefile
index 2cd1b1b..1a4c8db 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,7 @@ override LDFLAGS += \
 # Include our local Makefile.local first so that its first target is default
 include Makefile.local
 include lib/Makefile.local
+include checks/Makefile.local
 
 # And get user settings from the output of configure
 include Makefile.config
diff --git a/Makefile.local b/Makefile.local
index 1744747..f6ffd00 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -47,5 +47,8 @@ install-emacs: install emacs
 	install -m0644 notmuch.el $(DESTDIR)$(emacs_lispdir)
 	install -m0644 notmuch.elc $(DESTDIR)$(emacs_lispdir)
 
+check: checks/check
+	checks/check
+
 SRCS  := $(SRCS) $(notmuch_client_srcs)
 CLEAN := $(CLEAN) notmuch $(notmuch_client_modules) notmuch.elc notmuch.1.gz
diff --git a/checks/Makefile.local b/checks/Makefile.local
new file mode 100644
index 0000000..ec34d17
--- /dev/null
+++ b/checks/Makefile.local
@@ -0,0 +1,12 @@
+dir=checks
+extra_cflags += -I../lib
+
+check_c_srcs =		\
+	$(dir)/check.c
+
+check_modules = $(check_c_srcs:.c=.o)
+$(dir)/check: $(check_modules)
+	$(call quiet,CXX) $^ $(LDFLAGS) -o $@ lib/notmuch.a -lcheck
+
+SRCS  := $(SRCS) $(check_c_srcs)
+CLEAN := $(CLEAN) $(check_modules) $(dir)/check
diff --git a/checks/check.c b/checks/check.c
new file mode 100644
index 0000000..1b50de1
--- /dev/null
+++ b/checks/check.c
@@ -0,0 +1,90 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "notmuch-private.h"
+
+START_TEST (test_sha1_of_string)
+{
+   char *result;
+
+   result = notmuch_sha1_of_string("abcdefghi");
+   fail_if(strcmp("e1435dfb334ec81f2bdd5b0aa45969586e7681c0", result) != 0,
+	   "SHA1 results for notmuch_sha1_of_string do not match");
+}
+END_TEST
+
+START_TEST (test_sha1_of_file)
+{
+   char filename[20] = "";
+   char *result;
+   int fd;
+   int i;
+
+   strncpy (filename, "/tmp/notmuch.XXXXXX", sizeof (filename));
+   fd = mkstemp (filename);
+
+   fail_if (fd == -1,
+	    "Unable to create temporary file for SHA1 test");
+
+   for (i = 0; i < 13993; i++)
+      (void) write(fd, "\0", 1);
+
+   close(fd);
+
+   result = notmuch_sha1_of_file(filename);
+
+   unlink(filename);
+
+   fail_if (strcmp ("db38f0e9aa8eb5bcd0d73f3d1ed84d71712cc0ab", result) != 0,
+	    "SHA1 results for notmuch_sha1_of_file do not match");
+}
+END_TEST
+
+static Suite *
+notmuch_suite (void)
+{
+   Suite *s = suite_create ("NotMuch");
+   
+   /* SHA1 test cases */
+   TCase *tc_sha1 = tcase_create ("SHA1");
+   tcase_add_test (tc_sha1, test_sha1_of_string);
+   tcase_add_test (tc_sha1, test_sha1_of_file);
+   suite_add_tcase (s, tc_sha1);
+   return s;
+}
+
+int
+main (void)
+{
+   int number_failed;
+
+   Suite *s = notmuch_suite ();
+   SRunner *sr = srunner_create (s);
+
+   srunner_run_all (sr, CK_VERBOSE);
+   number_failed = srunner_ntests_failed (sr);
+   srunner_free (sr);
+
+   return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/configure b/configure
index e55f067..9e76f09 100755
--- a/configure
+++ b/configure
@@ -61,6 +61,14 @@ else
     have_valgrind=
 fi
 
+if pkg-config --exists check > /dev/null 2>&1; then
+    echo "Checking for the Check unit test framework... Yes."
+    have_check=1
+else
+    echo "Checking for the Check unit test framework... No."
+    have_check=
+fi
+
 if [ $errors -gt 0 ]; then
     cat <<EOF
 
-- 
1.6.5.2

[PATCH v3] Add the beginnings of a test suite.

[Jeffrey C. Ollie]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain, Size: 10336 bytes --]

This is the beginning of a test suite.  It uses the Check[1] unit
testing framework to handle the testing.  There are basic tests of the
SHA1 and tag manipulation routines, obviously many more will need to
be added.

Run "make check" from the toplevel directory to build and run the
checks.

[1] http://check.sourceforge.net/

Signed-off-by: Jeffrey C. Ollie <jeff@ocjtech.us>
---
 .gitignore                  |    1 +
 Makefile                    |    1 +
 Makefile.local              |    3 ++
 checks/Makefile.local       |   14 +++++++
 checks/notmuch-check-sha1.c |   76 ++++++++++++++++++++++++++++++++++++++
 checks/notmuch-check-tags.c |   84 +++++++++++++++++++++++++++++++++++++++++++
 checks/notmuch-check.c      |   40 ++++++++++++++++++++
 checks/notmuch-check.h      |   24 ++++++++++++
 configure                   |    8 ++++
 9 files changed, 251 insertions(+), 0 deletions(-)
 create mode 100644 checks/Makefile.local
 create mode 100644 checks/notmuch-check-sha1.c
 create mode 100644 checks/notmuch-check-tags.c
 create mode 100644 checks/notmuch-check.c
 create mode 100644 checks/notmuch-check.h

diff --git a/.gitignore b/.gitignore
index 8794354..bd542d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ notmuch.1.gz
 *~
 .*.swp
 *.elc
+checks/check
diff --git a/Makefile b/Makefile
index 2cd1b1b..1a4c8db 100644
--- a/Makefile
+++ b/Makefile
@@ -33,6 +33,7 @@ override LDFLAGS += \
 # Include our local Makefile.local first so that its first target is default
 include Makefile.local
 include lib/Makefile.local
+include checks/Makefile.local
 
 # And get user settings from the output of configure
 include Makefile.config
diff --git a/Makefile.local b/Makefile.local
index 1744747..f6ffd00 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -47,5 +47,8 @@ install-emacs: install emacs
 	install -m0644 notmuch.el $(DESTDIR)$(emacs_lispdir)
 	install -m0644 notmuch.elc $(DESTDIR)$(emacs_lispdir)
 
+check: checks/check
+	checks/check
+
 SRCS  := $(SRCS) $(notmuch_client_srcs)
 CLEAN := $(CLEAN) notmuch $(notmuch_client_modules) notmuch.elc notmuch.1.gz
diff --git a/checks/Makefile.local b/checks/Makefile.local
new file mode 100644
index 0000000..57797ea
--- /dev/null
+++ b/checks/Makefile.local
@@ -0,0 +1,14 @@
+dir=checks
+extra_cflags += -I$(dir)
+
+check_c_srcs =			\
+	$(dir)/notmuch-check.c		\
+	$(dir)/notmuch-check-sha1.c	\
+	$(dir)/notmuch-check-tags.c
+
+check_modules = $(check_c_srcs:.c=.o)
+$(dir)/check: $(check_modules)
+	$(call quiet,CXX) $^ $(LDFLAGS) -o $@ lib/notmuch.a -lcheck
+
+SRCS  := $(SRCS) $(check_c_srcs)
+CLEAN := $(CLEAN) $(check_modules) $(dir)/check
diff --git a/checks/notmuch-check-sha1.c b/checks/notmuch-check-sha1.c
new file mode 100644
index 0000000..c93fc6f
--- /dev/null
+++ b/checks/notmuch-check-sha1.c
@@ -0,0 +1,76 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+#include <string.h>
+
+#include "notmuch-private.h"
+#include "notmuch-check.h"
+
+START_TEST (test_sha1_of_string)
+{
+   char *result;
+
+   result = notmuch_sha1_of_string("abcdefghi");
+   fail_if(strcmp("e1435dfb334ec81f2bdd5b0aa45969586e7681c0", result) != 0,
+	   "SHA1 results for notmuch_sha1_of_string do not match");
+}
+END_TEST
+
+START_TEST (test_sha1_of_file)
+{
+   char filename[20] = "";
+   char *result;
+   int fd;
+   int i;
+
+   strncpy (filename, "/tmp/notmuch.XXXXXX", sizeof (filename));
+   fd = mkstemp (filename);
+
+   fail_if (fd == -1,
+	    "Unable to create temporary file for SHA1 test");
+
+   for (i = 0; i < 13993; i++)
+      (void) write(fd, "\0", 1);
+
+   close(fd);
+
+   result = notmuch_sha1_of_file(filename);
+
+   unlink(filename);
+
+   fail_if (strcmp ("db38f0e9aa8eb5bcd0d73f3d1ed84d71712cc0ab", result) != 0,
+	    "SHA1 results for notmuch_sha1_of_file do not match");
+}
+END_TEST
+
+Suite *
+notmuch_sha1_suite (void)
+{
+   Suite *s = suite_create ("NotMuch SHA1");
+   
+   /* SHA1 test cases */
+   TCase *tc_sha1 = tcase_create ("SHA1");
+   tcase_add_test (tc_sha1, test_sha1_of_string);
+   tcase_add_test (tc_sha1, test_sha1_of_file);
+   suite_add_tcase (s, tc_sha1);
+
+   return s;
+}
diff --git a/checks/notmuch-check-tags.c b/checks/notmuch-check-tags.c
new file mode 100644
index 0000000..5a3b5c7
--- /dev/null
+++ b/checks/notmuch-check-tags.c
@@ -0,0 +1,84 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "notmuch-private.h"
+#include "notmuch-check.h"
+
+START_TEST (test_tags_create)
+{
+   notmuch_tags_t *tags;
+
+   tags = _notmuch_tags_create(NULL);
+
+   fail_if (tags == NULL,
+	    "No tags structure was able to be allocated");
+
+   fail_if (notmuch_tags_has_more (tags),
+	    "Tags shouldn't have anything in it yet");
+
+   fail_unless (notmuch_tags_get (tags) == NULL,
+		"Shouldn't be able to get anything out of an empty tag set.");
+
+   notmuch_tags_destroy (tags);
+}
+END_TEST
+
+START_TEST (test_tags_sorting)
+{
+   notmuch_tags_t *tags;
+   char tag1[] = "1";
+   char tag2[] = "2";
+
+   tags = _notmuch_tags_create(NULL);
+
+   _notmuch_tags_add_tag(tags, tag2);
+   _notmuch_tags_add_tag(tags, tag1);
+
+   _notmuch_tags_prepare_iterator(tags);
+
+   fail_unless(strcmp (tag1, notmuch_tags_get(tags)) == 0,
+	       "Tags failed to be sorted properly.");
+
+   notmuch_tags_advance (tags);
+
+   fail_unless(strcmp (tag2, notmuch_tags_get(tags)) == 0,
+	       "Tags failed to be sorted properly.");
+
+   notmuch_tags_destroy (tags);
+}
+END_TEST
+
+Suite *
+notmuch_tags_suite (void)
+{
+   Suite *s = suite_create ("NotMuch Tags");
+   
+   /* Tags test cases */
+   TCase *tc_tags = tcase_create ("Tags");
+   tcase_add_test (tc_tags, test_tags_create);
+   tcase_add_test (tc_tags, test_tags_sorting);
+   suite_add_tcase (s, tc_tags);
+
+   return s;
+}
diff --git a/checks/notmuch-check.c b/checks/notmuch-check.c
new file mode 100644
index 0000000..378a391
--- /dev/null
+++ b/checks/notmuch-check.c
@@ -0,0 +1,40 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+
+#include "notmuch-private.h"
+#include "notmuch-check.h"
+
+int
+main (void)
+{
+   int number_failed;
+
+   SRunner *sr = srunner_create (NULL);
+   srunner_add_suite (sr, notmuch_sha1_suite ());
+   srunner_add_suite (sr, notmuch_tags_suite ());
+
+   srunner_run_all (sr, CK_VERBOSE);
+   number_failed = srunner_ntests_failed (sr);
+   srunner_free (sr);
+
+   return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/checks/notmuch-check.h b/checks/notmuch-check.h
new file mode 100644
index 0000000..f2ef07a
--- /dev/null
+++ b/checks/notmuch-check.h
@@ -0,0 +1,24 @@
+/* notmuch - Not much of an email program, (just index and search)
+ *
+ * Copyright © 2009 Jeffrey C. Ollie
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/ .
+ *
+ * Authors: Jeffrey C. Ollie <jeff@ocjtech.us>
+ */
+
+#include <check.h>
+
+Suite *notmuch_sha1_suite (void);
+Suite *notmuch_tags_suite (void);
diff --git a/configure b/configure
index e55f067..9e76f09 100755
--- a/configure
+++ b/configure
@@ -61,6 +61,14 @@ else
     have_valgrind=
 fi
 
+if pkg-config --exists check > /dev/null 2>&1; then
+    echo "Checking for the Check unit test framework... Yes."
+    have_check=1
+else
+    echo "Checking for the Check unit test framework... No."
+    have_check=
+fi
+
 if [ $errors -gt 0 ]; then
     cat <<EOF
 
-- 
1.6.5.2

Re: [PATCH v3] Add the beginnings of a test suite.

[Jameson Graef Rollins]

[-- Attachment #1: Type: text/plain, Size: 690 bytes --]

On Sat, Nov 28, 2009 at 03:33:14PM -0600, Jeffrey C. Ollie wrote:
> This is the beginning of a test suite.  It uses the Check[1] unit
> testing framework to handle the testing.  There are basic tests of the
> SHA1 and tag manipulation routines, obviously many more will need to
> be added.

Hey, Jeffrey.  I'm very glad to see someone starting to put together a
test suite.  They're usually a pain in the ass to put together, but
definitely a very very good thing to have.

> Run "make check" from the toplevel directory to build and run the
> checks.

I think it's usually standard to call the tests with "make test", yes?
Is there a reason to call them with "make check" instead?

jamie.

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 836 bytes --]

Re: [PATCH v3] Add the beginnings of a test suite.

[Jeffrey Ollie]

On Sun, Nov 29, 2009 at 8:08 AM, Jameson Graef Rollins
<jrollins@finestructure.net> wrote:
> On Sat, Nov 28, 2009 at 03:33:14PM -0600, Jeffrey C. Ollie wrote:
>> Run "make check" from the toplevel directory to build and run the
>> checks.
>
> I think it's usually standard to call the tests with "make test", yes?
> Is there a reason to call them with "make check" instead?

There's no special reason to call it "make check".  If that's the
consensus I'm willing to change it.

-- 
Jeff Ollie

Re: [PATCH] Add the beginnings of a test suite.

[Carl Worth]

[-- Attachment #1.1: Type: text/plain, Size: 2645 bytes --]

On Sat, 28 Nov 2009 10:02:03 -0600, "Jeffrey C. Ollie" <jeff@ocjtech.us> wrote:
> This is the beginning of a test suite.  It uses the Check[1] unit
> testing framework to handle the testing.  There's one basic test of
> the SHA1 routines, obviously many more will need to be added.

Hi Jeffrey,

Thanks very much for this contribution. I'm always highly in favor of
improving our testing infrastructure. And a good test suite was on of
the first items I put on the notmuch/TODO list.

However, it's not clear to me that this kind of C-based unit testing
will be able to easily support much of what I want from testing for
notmuch. It seems that most of the bugs I encounter, or features that I
want to test, require a lot of state to be setup that would be really
painful to do in C code.

For example, a recent bug I hit was:

	After moving an (old) existing file into the mail store,
	"notmuch new" won't pick it up.

And I don't see an easy way to test something like that from a typical
"check" test. Meanwhile, I can't see a lot of use from the kinds of
things that *could* be checked from a check test. We could easily check
little things like that after calling notmuch_message_add_tag that
notmuch_message_get_tags has the same tag in its list. But if we have to
do higher-level testing anyway, (for cases like the bug above), then
it's easy to cover fine-grained stuff like this there as well.

For the recent rename-support work I did I wrote the attached
notmuch-test script. It does the kind of high-level state setup I
described above, (and actually contains a test for precisely the bug I
described above).

I haven't added this to the repository yet since it needs a bunch of
work before being usable. Here are some things it needs:

  * Automated verification that tests are working, (currently I'm just
    manually verifying that the results are as expected---obviously
    that's not practical in the long term).

  * Modularization so that each test can be maintained as a small,
    independent snippet.

  * The ability to run an individual test in isolation without running
    the whole suite.

  * A fix in notmuch to get rid of the stupid sleep calls that are
    slowing down the current notmuch-test. (One approach would be to
    stop using mtime altogether. That might slow things down
    unacceptably in general use, but might be just fine for the test
    suite where the mail store can generally be quite tiny. Another
    approach would be to use a filesystem like ext4 with sub-second
    mtime support).

Anyway, just wanted to share some thoughts and some preliminary code for
a notmuch test suite.

-Carl


[-- Attachment #1.2: Type: application/pgp-signature, Size: 189 bytes --]

[-- Attachment #2: Lame prototype of a notmuch test suite --]
[-- Type: text/plain, Size: 4580 bytes --]

#!/bin/sh
set -e

find_notmuch_binary ()
{
    dir=$1

    while [ -n "$dir" ]; do
	bin=$dir/notmuch
	if [ -x $bin ]; then
	    echo $bin
	    return
	fi
	dir=$(dirname $dir)
	if [ "$dir" = "/" ]; then
	    break
	fi
    done

    echo notmuch
}

# Generate a new message in the mail directory, with
# a unique message ID and subject.
#
# The filename of the message generated is available as
# $gen_msg_filename
gen_msg_cnt=0
gen_msg_filename=""
generate_message ()
{
    gen_msg_cnt=$((gen_msg_cnt + 1))
    gen_msg_name=msg-$(printf "%03d" $gen_msg_cnt)

    if [ "$#" = "0" ]; then
	gen_msg_filename="${MAIL_DIR}/$gen_msg_name"
    else
	gen_msg_filename="${MAIL_DIR}/$1/$gen_msg_name"
	mkdir -p $(dirname $gen_msg_filename)
    fi

cat <<EOF >$gen_msg_filename
From: Notmuch Test Suite <test_suite@notmuchmail.org>
To: Notmuch Test Suite <test_suite@notmuchmail.org>
Message-Id: <msg-${gen_msg_cnt}@notmuch-test-suite>
Subject: Test message ${gen_msg_filename}
Date: Tue, 05 Jan 2010 15:43:57 -0800

This is just a test message at ${gen_msg_filename}
EOF
}

do_sleep ()
{
    sleep 1
}

TEST_DIR=$(pwd)/test.$$
MAIL_DIR=${TEST_DIR}/mail
export NOTMUCH_CONFIG=${TEST_DIR}/notmuch-config
NOTMUCH=$(find_notmuch_binary $(pwd))

rm -rf ${TEST_DIR}
mkdir ${TEST_DIR}
cd ${TEST_DIR}

mkdir ${MAIL_DIR}

cat <<EOF > ${NOTMUCH_CONFIG}
[database]
path=${MAIL_DIR}

[user]
name=Notmuch Test Suite
primary_email=test_suite@notmuchmail.org
EOF

echo "### Testing \"notmuch new\" with no messages"
$NOTMUCH new

echo "### Testing \"notmuch new\" with 1 new message"
do_sleep
generate_message
$NOTMUCH new

echo "### Testing \"notmuch new\" with 2 new messages"
do_sleep
generate_message
generate_message
$NOTMUCH new

echo "### Testing \"notmuch new\" with no new messages (and a non-empty database)"

$NOTMUCH new

echo "### Testing \"notmuch new\" with two new directories (one mail)"
rm -rf ${MAIL_DIR}/* ${MAIL_DIR}/.notmuch
mkdir ${MAIL_DIR}/def
mkdir ${MAIL_DIR}/ghi
generate_message def

$NOTMUCH new

echo "### Testing \"notmuch new\" with two new directories (one mail)---opposite inode order"

rm -rf ${MAIL_DIR}/.notmuch
mv ${MAIL_DIR}/ghi ${MAIL_DIR}/abc
rm ${MAIL_DIR}/def/*
generate_message abc

$NOTMUCH new

echo "### Testing \"notmuch new\" with 1 old message moved into the mail store"
rm -rf ${MAIL_DIR}/* ${MAIL_DIR}/.notmuch
generate_message
tmp_msg_filename=tmp/$gen_msg_filename
mkdir -p $(dirname $tmp_msg_filename)
mv $gen_msg_filename $tmp_msg_filename
do_sleep
$NOTMUCH new > /dev/null
do_sleep
mv $tmp_msg_filename $gen_msg_filename
$NOTMUCH new

echo "### Testing \"notmuch new\" with 1 renamed message"

do_sleep
generate_message
$NOTMUCH new > /dev/null
do_sleep
mv $gen_msg_filename ${gen_msg_filename}-renamed
$NOTMUCH new

echo "### Testing \"notmuch new\" with 1 deleted message"

do_sleep
rm ${gen_msg_filename}-renamed
$NOTMUCH new

echo "### Testing \"notmuch new\" with a new directory with 3 messages"

do_sleep
generate_message dir
generate_message dir
generate_message dir

$NOTMUCH new

echo "### Testing \"notmuch new\" with a renamed directory of 3 messages"

do_sleep
mv ${MAIL_DIR}/dir ${MAIL_DIR}/dir-renamed

$NOTMUCH new

echo "### Testing \"notmuch new\" with a deleted directory of 3 messages"

do_sleep
rm -rf ${MAIL_DIR}/dir-renamed

$NOTMUCH new

echo "### Testing \"notmuch new\" with a new directory with 3 messages (tail of list)"

do_sleep
generate_message zzz
generate_message zzz
generate_message zzz

$NOTMUCH new

echo "### Testing \"notmuch new\" with a deleted directory of 3 messages (tail of list)"

do_sleep
rm -rf ${MAIL_DIR}/zzz

$NOTMUCH new

echo "### Testing \"notmuch new\" with a symlink to an external directory of 1 message"

rm -rf ${MAIL_DIR}/.notmuch
mv ${MAIL_DIR} ${TEST_DIR}/actual_maildir

mkdir ${MAIL_DIR}
ln -s ${TEST_DIR}/actual_maildir ${MAIL_DIR}/symlink

$NOTMUCH new

echo "### Testing \"notmuch new\" with a symlink to an external file"
do_sleep
generate_message
external_msg_filename=${TEST_DIR}/external/$(basename $gen_msg_filename)
mkdir -p $(dirname $external_msg_filename)
mv $gen_msg_filename $external_msg_filename
ln -s $external_msg_filename $gen_msg_filename

$NOTMUCH new

echo "### Testing \"notmuch new\" with a two-level directory with 3 files"

do_sleep
generate_message two/levels
generate_message two/levels
generate_message two/levels

$NOTMUCH new

echo "### Testing \"notmuch new\" with deletion of two-level directory (3 files)"

do_sleep
rm -rf ${MAIL_DIR}/two

$NOTMUCH new

cat <<EOF
Notmuch test suite complete.

Intermediate state can be examined in:
	${TEST_DIR}
EOF