testsuite --recheck

[Ralf Wildenhues]

I find Automake's recheck very useful, so how about adding similar to
Autotest?

The sed script parsing the old testsuite.log file is a bit heuristic,
in that --version output of AT_TESTED programs could in principle mess
it up, but I regard that danger fairly low.  Oh well.

Thanks,
Ralf

    New Autotest testsuite option --recheck.
    
    * lib/autotest/general.m4 (AT_INIT): New variable $at_recheck.
    Escape hyphen in $at_dir early.  Accept command line switch
    --recheck.  Set $at_suite_log early, based on --directory
    switch; with --recheck, include the list of FAILed and XPASSed
    tests from old testsuite.log file in $at_groups.  Document
    --recheck in --help output.
    * tests/autotest.at (recheck): New test.
    * doc/autoconf.texi (testsuite Invocation): Document --recheck.
    * NEWS: Update.

diff --git a/NEWS b/NEWS
index dfe8cd7..f1fdefb 100644
--- a/NEWS
+++ b/NEWS
@@ -21,6 +21,9 @@ GNU Autoconf NEWS - User visible changes.
 
 ** AS_SET_CATFILE accepts nonliterals in its variable name argument now.
 
+** Autotest testsuites accept an option --recheck to rerun tests that
+   failed or passed unexpectedly during the last non-debug testsuite run.
+
 * Major changes in Autoconf 2.65 (2009-11-21) [stable]
   Released by Eric Blake, based on git versions 2.64.*.
 
diff --git a/doc/autoconf.texi b/doc/autoconf.texi
index 6be99aa..ca0d85c 100644
--- a/doc/autoconf.texi
+++ b/doc/autoconf.texi
@@ -23805,6 +23805,10 @@ testsuite Invocation
 the default for debugging scripts, but it can also be useful to debug
 the testsuite itself.
 
address@hidden --recheck
+Add to the selection all test groups that failed or passed unexpectedly
+during the last non-debugging test run.
+
 @item --trace
 @itemx -x
 Trigger shell tracing of the test groups.
diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4
index ac8f523..1e38abe 100644
--- a/lib/autotest/general.m4
+++ b/lib/autotest/general.m4
@@ -403,6 +403,8 @@ at_list_p=false
 at_clean=false
 # Test groups to run
 at_groups=
+# Whether to rerun failed tests.
+at_recheck=
 # Whether a write failure occurred
 at_write_fail=0
 
@@ -558,6 +560,9 @@ do
     --directory=* )
        at_change_dir=:
        at_dir=$at_optarg
+       if test x- = "x$at_dir" ; then
+         at_dir=./-
+       fi
        ;;
 
     # Parallel execution.
@@ -606,6 +611,9 @@ do
        `
        AS_VAR_APPEND([at_groups], ["$at_groups_selected "])
        ;;
+    --recheck)
+       at_recheck=:
+       ;;
 m4_divert_pop([PARSE_ARGS])dnl
 dnl Process *=* last to allow for user specified --option=* type arguments.
 m4_divert_push([PARSE_ARGS_END])dnl
@@ -633,10 +641,25 @@ done
 # Verify our last option didn't require an argument
 AS_IF([test -n "$at_prev"], [AS_ERROR([`$at_prev' requires an argument])])
 
+# The file containing the suite.
+at_suite_log=$at_dir/$as_me.log
+
 # Selected test groups.
-if test -z "$at_groups"; then
+if test -z "$at_groups$at_recheck"; then
   at_groups=$at_groups_all
 else
+  if test -n "$at_recheck" && test -r "$at_suite_log"; then
+    at_oldfails=`sed -n ['
+      /^Failed tests:$/,/^Skipped tests:$/{
+       s/^[ ]*\([1-9][0-9]*\):.*/\1/p
+      }
+      /^Unexpected passes:$/,/^## Detailed failed tests/{
+       s/^[ ]*\([1-9][0-9]*\):.*/\1/p
+      }
+      /^## Detailed failed tests/q
+      '] "$at_suite_log" | tr "$as_nl" ' '`
+    AS_VAR_APPEND([at_groups], ["$at_oldfails"])
+  fi
   # Sort the tests, removing duplicates.
   at_groups=`AS_ECHO(["$at_groups"]) | tr ' ' "$as_nl" | sort -nu`
 fi
@@ -686,6 +709,7 @@ Execution tuning:
   -k, --keywords=KEYWORDS
 [                 select the tests matching all the comma-separated KEYWORDS]
 [                 multiple \`-k' accumulate; prefixed \`!' negates a KEYWORD]
+      --recheck  select all tests that failed or passed unexpectedly last time
   -e, --errexit  abort as soon as a test fails; implies --debug
   -v, --verbose  force more detailed output
 [                 default for debugging scripts]
@@ -773,9 +797,6 @@ m4_divert_push([TESTS_BEGIN])dnl
 
 # Take any -C into account.
 if $at_change_dir ; then
-  if test x- = "x$at_dir" ; then
-    at_dir=./-
-  fi
   test x != "x$at_dir" && cd "$at_dir" \
     || AS_ERROR([unable to change directory])
   at_dir=`pwd`
@@ -840,7 +861,7 @@ m4_text_box([Directory structure.])
 # The directory the whole suite works in.
 # Should be absolute to let the user `cd' at will.
 at_suite_dir=$at_dir/$as_me.dir
-# The file containing the suite.
+# The file containing the suite ($at_dir might have changed since earlier).
 at_suite_log=$at_dir/$as_me.log
 # The directory containing helper files per test group.
 at_helper_dir=$at_suite_dir/at-groups
diff --git a/tests/autotest.at b/tests/autotest.at
index a09e2c1..ad38b7a 100644
--- a/tests/autotest.at
+++ b/tests/autotest.at
@@ -890,6 +890,52 @@ AT_CHECK([$CONFIG_SHELL ./micro-suite -C nonesuch --help], 
[0], [ignore], [])
 ])
 
 
+# --recheck.
+AT_CHECK_AT_TEST([recheck],
+  [AT_CHECK([:])
+   AT_CLEANUP
+   AT_SETUP([failing test])
+   AT_CHECK([exit 1])
+   AT_CLEANUP
+   AT_SETUP([xpassing test])
+   AT_XFAIL_IF([:])
+   AT_CHECK([:])
+   AT_CLEANUP
+   AT_SETUP([xfailing test])
+   AT_XFAIL_IF([:])
+   AT_CHECK([exit 1])
+], [], [], [], [], [], [
+AT_CHECK([$CONFIG_SHELL ./micro-suite --recheck], [0], [stdout])
+AT_CHECK([grep "0 tests were successful" stdout], [0], [ignore])
+
+AT_CHECK([$CONFIG_SHELL ./micro-suite], [1], [ignore], [ignore])
+AT_CHECK([grep 'only test' micro-suite.log], [0], [ignore])
+AT_CHECK([$CONFIG_SHELL ./micro-suite --recheck], [1], [ignore], [ignore])
+AT_CHECK([grep 'only test' micro-suite.log], [1])
+AT_CHECK([grep 'xfailing' micro-suite.log], [1])
+AT_CHECK([grep 'failing test' micro-suite.log], [0], [ignore])
+AT_CHECK([grep 'xpassing test' micro-suite.log], [0], [ignore])
+
+AT_CHECK([$CONFIG_SHELL ./micro-suite --clean], [0])
+AT_CHECK([test -f micro-suite.log], [1])
+
+dnl check specifying a different relative path to run in.
+AT_CHECK([mkdir sub1])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1], [1], [ignore], [ignore])
+AT_CHECK([test -f micro-suite.log], [1])
+AT_CHECK([test -f sub1/micro-suite.log], [0])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1 --recheck --list], [0], [stdout])
+AT_CHECK([grep 'only test' stdout], [1])
+AT_CHECK([grep 'xfailing test' stdout], [1])
+AT_CHECK([grep 'failing test' stdout], [0], [ignore])
+AT_CHECK([grep 'xpassing test' stdout], [0], [ignore])
+AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1 --recheck], [1], [ignore],
+        [ignore])
+AT_CHECK([grep 'failing test' sub1/micro-suite.log], [0], [ignore])
+AT_CHECK([grep 'xpassing test' sub1/micro-suite.log], [0], [ignore])
+], [1 4])
+
+
 ## -------- ##
 ## Banners. ##
 ## -------- ##