[Automake-ng] [PATCH 04/10] [ng] coverage: parallel-tests and dynamic $(TESTS) content

[Stefano Lattarini]

The recent changes to the parallel-tests implementation should allow us
to define or override TESTS with more dynamic contents; in particular,
something like:

  TESTS = $(wildcard t[0-9][0-9]*.sh)

should now work smoothly.  Let's cover this enhanced functionalities
explicitly in our testsuite.

* t/parallel-tests-dynamic.sh: New test.
* t/list-of-tests.mk: Add it.
* NG-NEWS: Document the new feature.

Signed-off-by: Stefano Lattarini <address@hidden>
---
 NG-NEWS                     |   14 +++--
 t/list-of-tests.mk          |    1 +
 t/parallel-tests-dynamic.sh |  146 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 157 insertions(+), 4 deletions(-)
 create mode 100755 t/parallel-tests-dynamic.sh

diff --git a/NG-NEWS b/NG-NEWS
index 62c666a..95b7262 100644
--- a/NG-NEWS
+++ b/NG-NEWS
@@ -79,10 +79,10 @@ Parallel testsuite harness
 * The new implementation of the parallel-tests harness tries to defer
   as much as possible processing on the $(TESTS) variable at make,
   rather than doing it at automake time.  This as some positive effects
-  (FIXME: describe and quantify them), but also entails a minor annoyance.
-  When automake runs, it cannot know anymore whether $(TESTS) contains
-  or not any entry with no recognized test suffix; as a consequence, it
-  always need the '$(LOG_DRIVER)' variable to be defined.  If it's not,
+  (described below), but also entails a minor annoyance.  When automake
+  runs, it cannot know anymore whether $(TESTS) contains or not any
+  entry with no recognized test suffix; as a consequence, it always
+  need the '$(LOG_DRIVER)' variable to be defined.  If it's not,
   automake will define it to a proper invocation of the 'test-driver'
   auxiliary script, and require that script to be present in the
   config-aux directory (or bring it in if "--add-missing" is in use).
@@ -92,6 +92,12 @@ Parallel testsuite harness
   configure.ac:
     AC_SUBST([LOG_DRIVER], ['unused but required by automake'])
 
+* The contents of $(TESTS) are mostly analyzed at make runtime rather
+  than at automake runtime, so that it is now possible to use GNU make
+  advanced features (like specific builtins or user-defined functions)
+  in the definition of $(TESTS); for example, the following should now
+  work as expected:
+    TESTS = $(wildcard $(srcdir)/t[0-9][0-9]*.sh)
 
 Miscellaneous
 =============
diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk
index 7d12d0f..e3febf1 100644
--- a/t/list-of-tests.mk
+++ b/t/list-of-tests.mk
@@ -772,6 +772,7 @@ t/parallel-tests-console-output.sh \
 t/parallel-tests-once.sh \
 t/parallel-tests-trailing-bslash.sh \
 t/parallel-tests-many.sh \
+t/parallel-tests-dynamic.sh \
 t/tests-environment.sh \
 t/am-tests-environment.sh \
 t/tests-environment-backcompat.sh \
diff --git a/t/parallel-tests-dynamic.sh b/t/parallel-tests-dynamic.sh
new file mode 100755
index 0000000..197046a
--- /dev/null
+++ b/t/parallel-tests-dynamic.sh
@@ -0,0 +1,146 @@
+#! /bin/sh
+# Copyright (C) 2012 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check that dynamic content for $(TESTS) and $(TEST_LOGS) is
+# supported.
+
+am_parallel_tests=yes
+. ./defs || Exit 1
+
+echo AC_OUTPUT >> configure.ac
+
+cat > ok <<'END'
+#!/bin/sh
+exit 0
+END
+
+cat > ko <<'END'
+#!/bin/sh
+exit 1
+END
+
+cat > er << 'END'
+#!/bin/sh
+echo $0 should not be run >&2
+exit 99
+END
+
+chmod a+x ko ok
+
+mkdir t
+cp ok t/nosuffix
+
+cp ok g1.sh
+cp ok g2.sh
+cp ok g3.sh
+cp ok g4.sh
+cp er g5.sh
+
+cp ok t00-foo.sh
+cp ok t02.sh
+cp ok t57_mu.sh
+cp ok t7311.sh
+cp ko t98S.sh
+cp ko t99.sh
+cp er t1.sh
+cp er t9.sh
+cp er tx98.sh
+
+cat > get-tests-list <<END
+#!/bin/sh
+echo "g1.sh  ${tab}g2.sh "
+if :; then echo '  g3.sh'; fi
+echo
+echo g4.sh
+END
+chmod a+x get-tests-list
+
+cat > Makefile.am << 'END'
+my_add_dirprefix = $(strip $(1))/$(strip $(2))
+EXTRA_DIST = $(TESTS) get-tests-list
+TEST_EXTENSIONS = .sh
+TESTS = $(wildcard t[0-9][0-9]*.sh) $(shell $(srcdir)/get-tests-list)
+TESTS += $(call my_add_dirprefix, t, nosuffix)
+XFAIL_TESTS = $(wildcard t9[0-9]*.sh)
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check > stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=11 pass=9 fail=0 xpass=0 xfail=2 skip=0 error=0
+
+grep '^PASS: t/nosuffix$' stdout
+grep '^PASS: g1\.sh$'     stdout
+grep '^PASS: g2\.sh$'     stdout
+grep '^PASS: g3\.sh$'     stdout
+grep '^PASS: g4\.sh$'     stdout
+grep '^PASS: t00-foo\.sh' stdout
+grep '^PASS: t02\.sh'     stdout
+grep '^PASS: t57_mu\.sh'  stdout
+grep '^PASS: t7311\.sh'   stdout
+grep '^XFAIL: t98S\.sh'   stdout
+grep '^XFAIL: t99\.sh'    stdout
+
+$MAKE mostlyclean
+test "`find . -name *.log`" = ./config.log
+
+$MAKE distcheck > stdout || { cat stdout; Exit 1; }
+cat stdout
+# FIXME: this is currently broken, as the $(wildcard ...) call miss the
+# FIXME: $(srcdir) component ...
+#count_test_results total=11 pass=9 fail=0 xpass=0 xfail=2 skip=0 error=0
+
+$MAKE check tests1='$(wildcard t00*.sh t98?.sh)' \
+            tests2='$(shell ./get-tests-list | sed 1d)' \
+            TESTS='$(tests1) $(tests2)' \
+  > stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=3 fail=0 xpass=0 xfail=1 skip=0 error=0
+
+grep '^PASS: g3\.sh$'     stdout
+grep '^PASS: g4\.sh$'     stdout
+grep '^PASS: t00-foo\.sh' stdout
+grep '^XFAIL: t98S\.sh'   stdout
+
+$MAKE mostlyclean
+test "`find . -name *.log`" = ./config.log
+
+# We can also override $(TEST_LOGS) dynamically.
+$MAKE check TEST_LOGS='$(shell echo t00 | sed "s/$$/-foo.log/") t99.log'
+test -f t00-foo.log
+test -f t99.log
+
+# A little tricky in that we rely on the .log files created by
+# the previous run to be present.
+$MAKE check TEST_LOGS="\
+  \$(wildcard t[0-9]*.log) \
+  \$(call my_add_dirprefix, t, nosuffix).log \
+" > stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=0
+grep '^PASS: t00-foo\.sh' stdout
+grep '^PASS: t/nosuffix'  stdout
+grep '^XFAIL: t99\.sh'    stdout
+
+:
-- 
1.7.9.5