[PATCHES v3] Allow custom testsuite drivers in Automake

[Stefano Lattarini]

Reference:
 <http://lists.gnu.org/archive/html/automake-patches/2011-05/msg00093.html>

Here is the new round of patches.  I've rebased them on latest master,
slighty edited the first patch (upating TESTS in Makefile.am, fixing a
bug in `test-driver-custom-no-pt-driver.test', adding a new test
`test-driver-fail.test', plus few cosmetic changes; see the attached
diff), and added a third patch that introduces support for test
protocols allowing for more than one test case per test script (as
TAP and SubUnit do).  This seems to work OK, but the new API (which
is described in the still-incomplete documentation extension) is a
little more confused and fragile than I'd like.  Criticism and
suggestions on how to improve it are welcome.

I'll soon post each updated patch in a distinct message in this thread.

Regards,
  Stefano

diff --git a/ChangeLog b/ChangeLog
index be12201..960abb6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-2011-06-03  Stefano Lattarini  <address@hidden>
+2011-06-04  Stefano Lattarini  <address@hidden>
 
        parallel-tests: allow custom driver scripts
        Allow suffix-based definition of custom "driver script" for the
@@ -22,10 +22,11 @@
        * tests/parallel-tests-no-pt-driver.test: New test.
        * tests/test-driver-custom.test: Likewise.
        * tests/test-driver-custom-xfail-tests.test: Likewise.
-       * tests/Makefile.am (TESTS): !!!TODO!!!
-       !!! TODO !!! ADD MORE TEST CASES.
+       * tests/test-driver-fail.test: Likewise.
+       * tests/Makefile.am: Update.
+       * NEWS: Update. TODO!
 
-2011-05-20  Stefano Lattarini  <address@hidden>
+2011-06-04  Stefano Lattarini  <address@hidden>
 
        parallel-tests: add auxiliary script 'pt-driver', refactor
        This refactoring should cause no API of functionality change,
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 8efefab..36445b5 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -719,6 +719,10 @@ parallel-tests-subdir.test \
 parallel-tests-interrupt.test \
 parallel-tests-reset-term.test \
 parallel-tests-pt-driver.test \
+test-driver-custom-no-pt-driver.test \
+test-driver-custom.test \
+test-driver-custom-xfail-tests.test \
+test-driver-fail.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 85e20cb..df2a818 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -973,6 +973,10 @@ parallel-tests-subdir.test \
 parallel-tests-interrupt.test \
 parallel-tests-reset-term.test \
 parallel-tests-pt-driver.test \
+test-driver-custom-no-pt-driver.test \
+test-driver-custom.test \
+test-driver-custom-xfail-tests.test \
+test-driver-fail.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/test-driver-custom-no-pt-driver.test 
b/tests/test-driver-custom-no-pt-driver.test
index 59fda92..f4c996e 100755
--- a/tests/test-driver-custom-no-pt-driver.test
+++ b/tests/test-driver-custom-no-pt-driver.test
@@ -24,7 +24,7 @@ parallel_tests=yes
 cat >> configure.in <<'END'
 AC_PROG_CC
 AC_CONFIG_FILES([sub1/Makefile sub2/Makefile])
-AC_SUBST([X_LOG_WRAPPER], [none])
+AC_SUBST([X_LOG_DRIVER], [none])
 AC_OUTPUT
 END
 
diff --git a/tests/test-driver-custom-xfail-tests.test 
b/tests/test-driver-custom-xfail-tests.test
index e50b2a5..0d10594 100755
--- a/tests/test-driver-custom-xfail-tests.test
+++ b/tests/test-driver-custom-xfail-tests.test
@@ -28,7 +28,7 @@ END
 
 cat > Makefile.am <<'END'
 SUBDIRS = . sub1 sub2
-TEST_LOG_DRIVER = $(srcdir)/tw
+TEST_LOG_DRIVER = $(srcdir)/td
 TESTS = pass.test xfail.test
 XFAIL_TESTS = xfail.test
 END
@@ -38,7 +38,7 @@ mkdir sub1 sub2
 cat > sub1/Makefile.am <<END
 empty =
 
-TEST_LOG_DRIVER = \$(top_srcdir)/tw
+TEST_LOG_DRIVER = \$(top_srcdir)/td
 
 # XFAIL_TESTS should gracefully handle TAB characters, and multiple
 # whitespaces.
@@ -51,7 +51,7 @@ END
 cat > sub2/Makefile.am <<'END'
 AUTOMAKE_OPTIONS = -Wno-portability-recursive
 
-TEST_LOG_DRIVER = $(srcdir)/../tw
+TEST_LOG_DRIVER = $(srcdir)/../td
 
 # XFAIL_TESTS should gracefully AC_SUBST @substitution@ and
 # make variables indirections.
@@ -86,7 +86,7 @@ for i in 1 2 3 4 5 6; do
   cp xfail.test sub2/x$i.test
 done
 
-cat > tw <<'END'
+cat > td <<'END'
 #! /bin/sh
 set -e
 test_name=INVALID
@@ -118,7 +118,7 @@ case $st,$expect_failure in
   *) echo "UNEXPECTED OUTCOME: $test_name"; exit 99;;
 esac | tee "$log_file"
 END
-chmod a+x tw
+chmod a+x td
 
 $ACLOCAL
 $AUTOCONF
@@ -135,7 +135,8 @@ for dir in sub1 sub2; do
   cd $dir
   cp pass.test x1.test
   cp x2.test pass.test
-  VERBOSE=yes $MAKE check && Exit 1
+  $MAKE check && { cat test-suite.log; Exit 1; }
+  cat test-suite.log
   grep '^FAIL: pass\.test$' test-suite.log
   grep '^XPASS: x1\.test$' test-suite.log
   test `grep -c '^FAIL' test-suite.log` -eq 1
diff --git a/tests/test-driver-fail.test b/tests/test-driver-fail.test
new file mode 100755
index 0000000..e1d2f9c
--- /dev/null
+++ b/tests/test-driver-fail.test
@@ -0,0 +1,65 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: what happens when a test driver fails?  Well,
+# "make check" should at least fail too, and the test-suite.log
+# shouldn't be created.  Unfortunately, we cannot truly control also
+# the (non-)creation of individual test logs, since those are expected
+# to be created by the drivers themselves, and an ill-behaved driver
+# (like our dummy one in this test) might leave around a test log even
+# in case of internal failures.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TEST_LOG_DRIVER = ./oops
+TESTS = foo.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# The testsuite driver does not exist.
+$MAKE check && Exit 1
+test ! -f test-suite.log
+
+# The testsuite driver exists and create the test log files, but fails.
+
+cat > oops <<'END'
+#!/bin/sh
+: > foo.log
+echo 'Oops, I fail!' >&2
+exit 1
+END
+chmod a+x oops
+
+$MAKE check && Exit 1
+test ! -f test-suite.log
+
+: