[FYI 1/2] {prove} tests: allow use of `prove' to run automake testsuite

[Stefano Lattarini]

* tests/prove-runner: New helper script; mostly it wraps
our `*.test' test scripts to make them runnable by the
`prove' utility.
* tests/Makefile.am (EXTRA_DIST): Distribute it.
(PROVE, AM_PROVEFLAGS, AM_PROVECMD): New variables.
(prove, installprove): New targets, used run the automake
testsuite through the `prove' utility.
* tests/gen-testsuite-part: For any considered dependency,
generate a variables that groups all the tests depending on
it.  This is especially useful for use in the new `prove'
and `installprove' targets, which cannot rely on make for
inter-test dependencies, and have to handle them "by hand".
---
 ChangeLog                     |   16 ++++
 tests/Makefile.am             |   83 +++++++++++++++++++++
 tests/Makefile.in             |  159 +++++++++++++++++++++++++++++++++++++++--
 tests/gen-testsuite-part      |   13 ++--
 tests/prove-runner            |   50 +++++++++++++
 5 files changed, 311 insertions(+), 10 deletions(-)
 create mode 100755 tests/prove-runner
 mode change 100644 => 100755 tests/test-logs-repeated.test

diff --git a/ChangeLog b/ChangeLog
index 39aa991..f4aa951 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+2011-09-10  Stefano Lattarini  <address@hidden>
+
+       tests: allow use of `prove' to run automake testsuite
+       * tests/prove-runner: New helper script; mostly it wraps
+       our `*.test' test scripts to make them runnable by the
+       `prove' utility.
+       * tests/Makefile.am (EXTRA_DIST): Distribute it.
+       (PROVE, AM_PROVEFLAGS, AM_PROVECMD): New variables.
+       (prove, installprove): New targets, used run the automake
+       testsuite through the `prove' utility.
+       * tests/gen-testsuite-part: For any considered dependency,
+       generate a variables that groups all the tests depending on
+       it.  This is especially useful for use in the new `prove'
+       and `installprove' targets, which cannot rely on make for
+       inter-test dependencies, and have to handle them "by hand".
+
 2011-09-09  Stefano Lattarini  <address@hidden>
 
        test defs: more environment cleanup
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 51edc2f..1f4f9ab 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -183,3 +183,86 @@ clean-local-check:
           find "$$@" -type d ! -perm -700 -exec chmod u+rwx {} ';'; \
           rm -rf "$$@"; \
         fi;
+
+## Run the testsuite through the `prove' utility.  Mostly for maintainers,
+## and to find out possible incompatibilities between `prove' and the
+## Automake own TAP driver.
+
+PROVE = prove
+AM_PROVEFLAGS = --timer
+AM_PROVECMD = \
+  test -z "$$tests" || \
+    { \
+      $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
+      $(PROVE) $(AM_PROVEFLAGS) -e $(srcdir)/prove-runner $(PROVEFLAGS) \
+      $$tests $(AM_TESTS_FD_REDIRECT); \
+    }
+
+EXTRA_DIST += prove-runner
+
+.PHONY: prove installprove
+prove installprove: $(TESTS) prove-runner defs-static
+       @exit_status=0; \
+## The testsuite can be run either using the automake build tree or
+## the already-installed automake package.
+       case '$@' in \
+         install*) am_running_installcheck=yes;; \
+         *) am_running_installcheck=no;; \
+       esac; \
+       export am_running_installcheck; \
+## XXX Our poor-man way to handle inter-tests dependencies without the
+## XXX help of make.  This is obviously fragile and doesn't scale,
+## XXX but for now it's good enough since we have only one possible
+## XXX inter-tests dependency at the moment.
+       need_tap_common_setup=no; \
+       lst=' $(TESTS) '; for t in $$lst; do \
+         case ' $(on_tap_with_common_setup_TESTS) ' in \
+           *" $$t "*) need_tap_common_setup=yes; break;; \
+         esac; \
+       done; \
+## For VPATH builds.
+       if test $$need_tap_common_setup = yes; then \
+         t=tap-common-setup.test; \
+         if test -f ./$$t; then \
+           tests=./$$t; \
+         else \
+           tests=$(srcdir)/$$t; \
+         fi; \
+## Do not stop if there's a failure here, we want to run the following
+## tests anyway.
+         echo "$@: running pre-requisite tests ..."; \
+         $(AM_PROVECMD) || exits_status=$$?; \
+         echo "$@: done.  We'll shortly run the main testsuite."; \
+       fi; \
+## Handle VPATH builds, and ensure that the tests have a properly
+## qualified path.
+       tests=`\
+         for t in $(TESTS); do \
+## XXX In next line, another hack for inter-tests dependencies: this
+## XXX can't run concurrently to the tests that use the data files it
+## XXX generates.
+           case "$$need_tap_common_setup,/$$t" in \
+             yes,*/tap-common-setup.test) continue;; \
+           esac; \
+           if test -f ./$$t; then \
+             echo ./$$t; \
+           else \
+             echo $(srcdir)/$$t; \
+           fi; \
+         done`; \
+## Extra environment, mostly for use by `prove-runner'.
+## XFAIL_TESTS gets normalized to play nicer with VPATH builds.
+       XFAIL_TESTS=`\
+         for t in $(XFAIL_TESTS); do \
+           echo $$t; echo ./$$t; echo $(srcdir)/$$t; \
+         done | LC_ALL=C sort | uniq`; \
+       XFAIL_TESTS=`echo $$XFAIL_TESTS`; \
+       TAP_RUNNER='$(TAP_LOG_COMPILE)' \
+       TEST_RUNER='$(TEST_LOG_COMPILE)' \
+       export XFAIL_TESTS TAP_RUNNER TEST_RUNNER; \
+## For consistency with the Automake testsuite harness, that exports
+## this unconditionally in the test scripts' environment.
+       srcdir='$(srcdir)'; export srcdir; \
+## Here we go.
+       $(AM_PROVECMD) || exit_status=$$?; \
+       exit $$exit_status;
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 5ae15b1..b131ecf 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -295,11 +295,12 @@ TAP_LOG_COMPILER = $(LOG_COMPILER)
 TAP_LOG_DRIVER = AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/lib/tap-driver.sh
 AM_TAP_LOG_DRIVER_FLAGS = --merge --comments --diagnostic-string \
        `printf '\043%%\043\n'`
-EXTRA_DIST = ChangeLog-old $(handwritten_TESTS) tap-summary-aux.sh \
-       tap-setup.sh extract-testsuite-summary \
+EXTRA_DIST = ChangeLog-old $(handwritten_TESTS) tap-setup.sh \
+       tap-summary-aux.sh extract-testsuite-summary \
        testsuite-summary-checks.sh trivial-test-driver \
        $(generated_TESTS) gen-testsuite-part tap-functions.sh \
-       plain-functions.sh distcheck-hook-m4.am defs-static.in
+       plain-functions.sh distcheck-hook-m4.am defs-static.in \
+       prove-runner
 
 # Hand-written tests.
 
@@ -1435,6 +1436,96 @@ yflags-force-override.test \
 yflags-force-conditional.test \
 yflags-var-expand.test
 
+on_tap_with_common_setup_TESTS = \
+  tap-ambiguous-directive.test \
+  tap-autonumber.test \
+  tap-bailout.test \
+  tap-bailout-and-logging.test \
+  tap-bailout-suppress-badexit.test \
+  tap-bailout-suppress-later-diagnostic.test \
+  tap-bailout-suppress-later-errors.test \
+  tap-color.test \
+  tap-deps.test \
+  tap-diagnostic.test \
+  tap-empty-diagnostic.test \
+  tap-empty.test \
+  tap-escape-directive.test \
+  tap-escape-directive-2.test \
+  tap-exit.test \
+  tap-signal.tap \
+  tap-fancy.test \
+  tap-fancy2.test \
+  tap-global-log.test \
+  tap-global-result.test \
+  tap-html.test \
+  tap-log.test \
+  tap-msg0-result.test \
+  tap-msg0-directive.test \
+  tap-msg0-planskip.test \
+  tap-msg0-bailout.test \
+  tap-msg0-misc.test \
+  tap-merge-stdout-stderr.test \
+  tap-no-merge-stdout-stderr.test \
+  tap-no-disable-hard-error.test \
+  tap-no-spurious-summary.test \
+  tap-no-spurious-numbers.test \
+  tap-no-spurious.test \
+  tap-not-ok-skip.test \
+  tap-number-wordboundary.test \
+  tap-numeric-description.test \
+  tap-negative-numbers.test \
+  tap-numbers-leading-zero.test \
+  tap-out-of-order.test \
+  tap-passthrough.test \
+  tap-passthrough-exit.test \
+  tap-plan.test \
+  tap-plan-corner.test \
+  tap-plan-errors.test \
+  tap-plan-middle.test \
+  tap-plan-whitespace.test \
+  tap-plan-leading-zero.test \
+  tap-plan-malformed.test \
+  tap-missing-plan-and-bad-exit.test \
+  tap-planskip.test \
+  tap-planskip-late.test \
+  tap-planskip-and-logging.test \
+  tap-planskip-unplanned.test \
+  tap-planskip-unplanned-corner.test \
+  tap-planskip-case-insensitive.test \
+  tap-planskip-whitespace.test \
+  tap-planskip-badexit.test \
+  tap-planskip-bailout.test \
+  tap-planskip-later-errors.test \
+  tap-realtime.test \
+  tap-test-number-0.test \
+  tap-recheck-logs.test \
+  tap-result-comment.test \
+  tap-todo-skip-together.test \
+  tap-todo-skip-whitespace.test \
+  tap-todo-skip.test \
+  tap-unplanned.test \
+  tap-whitespace-normalization.test \
+  tap-with-and-without-number.test \
+  tap-xfail-tests.test
+
+check_tap_testsuite_summary_TESTS = \
+  tap-summary.test \
+  tap-summary-color.test
+
+extract_testsuite_summary_TESTS = \
+  testsuite-summary-count-many.test
+
+check_testsuite_summary_TESTS = \
+  testsuite-summary-color.test \
+  testsuite-summary-count.test
+
+use_trivial_test_driver_TESTS = \
+  testsuite-summary-count-many.test \
+  test-driver-custom-multitest.test \
+  test-driver-custom-multitest-recheck.test \
+  test-driver-custom-multitest-recheck2.test \
+  test-driver-custom-html.test
+
 generated_TESTS = ar-lib-w.test compile-w.test compile2-w.test \
        compile3-w.test compile4-w.test compile5-w.test \
        compile6-w.test instsh2-w.test instsh3-w.test mdate5-w.test \
@@ -1527,6 +1618,16 @@ CLEANFILES = defs-static
 am__tmk = tests-in-makefile-list.tmp
 am__tfs = tests-on-filesystem-list.tmp
 am__tdf = diff-in-tests-lists.tmp
+PROVE = prove
+AM_PROVEFLAGS = --timer
+AM_PROVECMD = \
+  test -z "$$tests" || \
+    { \
+      $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
+      $(PROVE) $(AM_PROVEFLAGS) -e $(srcdir)/prove-runner $(PROVEFLAGS) \
+      $$tests $(AM_TESTS_FD_REDIRECT); \
+    }
+
 all: all-am
 
 .SUFFIXES:
@@ -1920,8 +2021,6 @@ uninstall-am:
 print-list-of-tests:
        @echo $(handwritten_TESTS)
 .PHONY: print-list-of-tests
-tap-summary.log: tap-summary-aux.sh 
-tap-summary-color.log: tap-summary-aux.sh 
 tap-ambiguous-directive.log: tap-setup.sh tap-common-setup.log
 tap-autonumber.log: tap-setup.sh tap-common-setup.log
 tap-bailout.log: tap-setup.sh tap-common-setup.log
@@ -1992,6 +2091,8 @@ tap-unplanned.log: tap-setup.sh tap-common-setup.log
 tap-whitespace-normalization.log: tap-setup.sh tap-common-setup.log
 tap-with-and-without-number.log: tap-setup.sh tap-common-setup.log
 tap-xfail-tests.log: tap-setup.sh tap-common-setup.log
+tap-summary.log: tap-summary-aux.sh 
+tap-summary-color.log: tap-summary-aux.sh 
 testsuite-summary-count-many.log: extract-testsuite-summary 
 testsuite-summary-color.log: testsuite-summary-checks.sh 
 testsuite-summary-count.log: testsuite-summary-checks.sh 
@@ -2341,6 +2442,54 @@ clean-local-check:
           rm -rf "$$@"; \
         fi;
 
+.PHONY: prove installprove
+prove installprove: $(TESTS) prove-runner defs-static
+       @exit_status=0; \
+       case '$@' in \
+         install*) am_running_installcheck=yes;; \
+         *) am_running_installcheck=no;; \
+       esac; \
+       export am_running_installcheck; \
+       need_tap_common_setup=no; \
+       lst=' $(TESTS) '; for t in $$lst; do \
+         case ' $(on_tap_with_common_setup_TESTS) ' in \
+           *" $$t "*) need_tap_common_setup=yes; break;; \
+         esac; \
+       done; \
+       if test $$need_tap_common_setup = yes; then \
+         t=tap-common-setup.test; \
+         if test -f ./$$t; then \
+           tests=./$$t; \
+         else \
+           tests=$(srcdir)/$$t; \
+         fi; \
+         echo "$@: running pre-requisite tests ..."; \
+         $(AM_PROVECMD) || exits_status=$$?; \
+         echo "$@: done.  We'll shortly run the main testsuite."; \
+       fi; \
+       tests=`\
+         for t in $(TESTS); do \
+           case "$$need_tap_common_setup,/$$t" in \
+             yes,*/tap-common-setup.test) continue;; \
+           esac; \
+           if test -f ./$$t; then \
+             echo ./$$t; \
+           else \
+             echo $(srcdir)/$$t; \
+           fi; \
+         done`; \
+       XFAIL_TESTS=`\
+         for t in $(XFAIL_TESTS); do \
+           echo $$t; echo ./$$t; echo $(srcdir)/$$t; \
+         done | LC_ALL=C sort | uniq`; \
+       XFAIL_TESTS=`echo $$XFAIL_TESTS`; \
+       TAP_RUNNER='$(TAP_LOG_COMPILE)' \
+       TEST_RUNER='$(TEST_LOG_COMPILE)' \
+       export XFAIL_TESTS TAP_RUNNER TEST_RUNNER; \
+       srcdir='$(srcdir)'; export srcdir; \
+       $(AM_PROVECMD) || exit_status=$$?; \
+       exit $$exit_status;
+
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.
 .NOEXPORT:
diff --git a/tests/gen-testsuite-part b/tests/gen-testsuite-part
index fc21487..1c1c55b 100755
--- a/tests/gen-testsuite-part
+++ b/tests/gen-testsuite-part
@@ -158,7 +158,7 @@ my %deps_extractor =
         line_matcher => qr/\btap-summary-aux\.sh\b/,
         dist_prereqs => "tap-summary-aux.sh",
       },
-    is_test_on_tap_with_common_setup =>
+    on_tap_with_common_setup =>
       {
         line_matcher => qr/\btap-setup\.sh\b/,
         dist_prereqs => "tap-setup.sh",
@@ -246,7 +246,7 @@ my %test_generators =
 
 parse_options @ARGV;
 
-my @tests = get_list_of_tests;
+my @all_tests = get_list_of_tests;
 
 print "## -*- Makefile -*-\n";
 print "## Generated by $me.  DO NOT EDIT BY HAND!\n\n";
@@ -263,9 +263,12 @@ while (my ($k, $x) = each %deps_extractor)
   {
     my $dist_prereqs = $x->{dist_prereqs} || "";
     my $nodist_prereqs = $x->{nodist_prereqs} || "";
-    my @logs = grep { line_match $x->{line_matcher}, $_ } @tests;
-    map { s/\.[^.]*$//; s/$/\.log/; } @logs;
+    my @tests = grep { line_match $x->{line_matcher}, $_ } @all_tests;
+    map { s/\.[^.]*$//; s/$/\.log/; } (my @logs = @tests);
     print "## Added by deps-extracting key `$k'.\n";
+    ## The list of all tests which have a dependency detected by the
+    ## current key.
+    print join(" \\\n  ", "${k}_TESTS =", @tests) . "\n";
     print "EXTRA_DIST += $dist_prereqs\n";
     map { print "$_: $dist_prereqs $nodist_prereqs\n" } @logs;
     print "\n";
@@ -289,7 +292,7 @@ while (my ($k, $g) = each %test_generators)
     my @wrapped_tests = grep {
       line_match ($g->{line_matcher}, $_)
         && !line_match ($g->{line_rejecter}, $_)
-    } @tests;
+    } @all_tests;
     foreach my $wrapped_test (@wrapped_tests)
       {
         (my $base = $wrapped_test) =~ s/\.([^.]*)$//;
diff --git a/tests/prove-runner b/tests/prove-runner
new file mode 100755
index 0000000..41c4c52
--- /dev/null
+++ b/tests/prove-runner
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Helper script to run the automake testsuite through the `prove' utility.
+# See the "prove" and "installprove" targets in tests/Makefile.am.
+
+set -u
+
+nl='
+'
+
+tst=$1
+shift
+
+case $tst in
+  *.test)
+    echo 1..1
+    # Protect the output contents of the test script, since it might
+    # sometimes contain TAP lines that would confuse `prove'.  This
+    # This has already happened in practice, especially for the tests
+    # checking TAP support in automake-generated test harnesses.
+    (${TEST_RUNNER-} "$tst" ${1+"$@"}; echo exit status: $?) 2>&1 | awk '
+      BEGIN { lastine = "exit status: 255"; }
+      { print " " $0; lastline = $0; }
+      END { st = lastline; sub("^exit status: *", "", st); exit st; }
+    '
+    st=$?
+    if test $st -eq 99; then
+      echo "not ok - $tst: hard failure (exit status $st)"
+    elif test $st -eq 77; then
+      echo "ok - $tst # SKIP"
+    else
+      case " ${XFAIL_TESTS-} " in
+        *" $tst "*) xs=' # TODO';;
+        *) xs='';;
+      esac
+      if test $st -eq 0; then
+        echo "ok - test script $tst passed$xs"
+      else
+        echo "not ok - test script $tst failed (exit status $st)$xs"
+      fi
+    fi
+    exit 0
+    ;;
+  *.tap)
+    ${TAP_RUNNER-} "$tst" ${1+"$@"} 2>&1
+    exit $?
+    ;;
+esac
+
+echo "Bail out! Invalid test script name '$tst'"
+exit 99
diff --git a/tests/test-logs-repeated.test b/tests/test-logs-repeated.test
old mode 100644
new mode 100755
-- 
1.7.2.3