# # # patch "Makefile.am" # from [47b547567f27ecaed8467451a54d3bfbca25be8d] # to [9c3489048d11104d7f82f84db5204d060b313cb5] # # patch "tester-tests/cleanup-1/__driver__.lua" # from [d16b6e520060842d1cf59e441e4b0c6f0f1f3302] # to [146d477bf5223d9963adc2f0fcaf47916f5326e6] # # patch "tester-tests/cleanup-2/__driver__.lua" # from [6438a8657a56d840d49cd65d44ceca2cb42d7597] # to [652dbe860039c60ddc896417c02508f4d7488ff9] # # patch "tester.cc" # from [359d6329c9f549d6d0d47ef66fa24b7ac604255d] # to [7f55feff516d38f2795746aa64c4d742e52bb6aa] # # patch "testlib.lua" # from [8c72b168db4237136c272bded45474b1ca43da38] # to [df3aee450ca03e7454704cde586448214cb6e2c5] # # patch "testsuite.lua" # from [9ca3325fc516d2e6235f800c444e1152562d8eae] # to [0ae9af3c12cea3f293bd302fde0c54c4bfa45a38] # ============================================================ --- Makefile.am 47b547567f27ecaed8467451a54d3bfbca25be8d +++ Makefile.am 9c3489048d11104d7f82f84db5204d060b313cb5 @@ -338,7 +338,7 @@ tester_LDADD = $(addprefix mtn-, $(patsu tester_SOURCES = tester.cc nodist_tester_SOURCES = testlib.c tester_LDADD = $(addprefix mtn-, $(patsubst %.cc, %.$(OBJEXT), \ - $(filter %.cc, $(SANITY_CORE_SOURCES) $(LUAEXT_SOURCES)))) + $(filter %.cc, $(SANITY_CORE_SOURCES) $(LUAEXT_SOURCES) option.cc))) txt2c_SOURCES = txt2c.cc ============================================================ --- tester-tests/cleanup-1/__driver__.lua d16b6e520060842d1cf59e441e4b0c6f0f1f3302 +++ tester-tests/cleanup-1/__driver__.lua 146d477bf5223d9963adc2f0fcaf47916f5326e6 @@ -1,6 +1,10 @@ +-- no state whatsoever is inherited across tests +-- (see cleanup-2 for the other half of this test) function cleanup() - -- stuff under test isn't cleaned between tests; - -- only predefined vars get reset - test.cleanup_ran = true + cleanup_ran = true + test.cleanup_ran = true end + +t_ran = true +test.t_ran = true ============================================================ --- tester-tests/cleanup-2/__driver__.lua 6438a8657a56d840d49cd65d44ceca2cb42d7597 +++ tester-tests/cleanup-2/__driver__.lua 652dbe860039c60ddc896417c02508f4d7488ff9 @@ -1 +1,5 @@ -check(test.cleanup_ran == true) +-- the variables set by cleanup-1 should not have survived to this point +check(t_ran == nil) +check(cleanup_ran == nil) +check(test.t_ran == nil) +check(test.cleanup_ran == nil) ============================================================ --- tester.cc 359d6329c9f549d6d0d47ef66fa24b7ac604255d +++ tester.cc 7f55feff516d38f2795746aa64c4d742e52bb6aa @@ -6,6 +6,7 @@ #include "lua.hh" #include "platform.hh" #include "sanity.hh" +#include "option.hh" #include #include @@ -36,6 +37,12 @@ #include #endif +#ifdef WIN32 +#define NULL_DEVICE "NUL:" +#else +#define NULL_DEVICE "/dev/null" +#endif + // defined in testlib.c, generated from testlib.lua extern char const testlib_constant[]; @@ -312,8 +319,11 @@ map orig_env_vars; map orig_env_vars; -string source_dir; -string run_dir; +static string argv0; +static string firstdir; +static string source_dir; +static string run_dir; +static string testfile; static int panic_thrower(lua_State * st) { @@ -484,26 +494,6 @@ LUAEXT(posix_umask, ) #endif } -LUAEXT(go_to_test_dir, ) -{ - try - { - string tname = basename(luaL_checkstring(L, -1)); - string testdir = run_dir + "/" + tname; - do_remove_recursive(testdir); - do_mkdir(testdir); - change_current_working_dir(testdir); - lua_pushstring(L, testdir.c_str()); - lua_pushstring(L, tname.c_str()); - return 2; - } - catch(informative_failure & e) - { - lua_pushnil(L); - return 1; - } -} - LUAEXT(chdir, ) { try @@ -520,24 +510,6 @@ LUAEXT(chdir, ) } } -LUAEXT(clean_test_dir, ) -{ - try - { - string tname = basename(luaL_checkstring(L, -1)); - string testdir = run_dir + "/" + tname; - change_current_working_dir(run_dir); - do_remove_recursive(testdir); - lua_pushboolean(L, true); - return 1; - } - catch(informative_failure & e) - { - lua_pushnil(L); - return 1; - } -} - LUAEXT(remove_recursive, ) { try @@ -588,21 +560,6 @@ LUAEXT(copy_recursive, ) } } -LUAEXT(leave_test_dir, ) -{ - try - { - change_current_working_dir(run_dir); - lua_pushboolean(L, true); - return 1; - } - catch(informative_failure & e) - { - lua_pushnil(L); - return 1; - } -} - LUAEXT(mkdir, ) { try @@ -824,6 +781,78 @@ LUAEXT(require_not_root, ) "Please try again with a normal user account.\n")); exit(1); } + return 0; +} + +// run_tests_in_children (to_run, reporter) + +// Run all of the tests in TO_RUN, each in its own isolated directory and +// child process. As each exits, call REPORTER with the test number and +// name, and the exit status. If REPORTER returns true, delete the test +// directory, otherwise leave it alone. The point of this exercise is to +// isolate in one C function all the black art involved with safe creation +// and destruction of child processes and their context. +LUAEXT(run_tests_in_children, ) +{ + // lua arguments + const int to_run = 1; + const int reporter = 2; + + if (lua_gettop(L) != 2) + return luaL_error(L, "wrong number of arguments"); + + luaL_argcheck(L, lua_istable(L, 1), 1, "expected a table"); + luaL_argcheck(L, lua_isfunction(L, 2), 2, "expected a function"); + + // iterate over to_run... + char const * argv[6]; + lua_pushnil(L); + while (lua_next(L, to_run) != 0) + { + luaL_checkinteger(L, -2); + string testname = luaL_checkstring(L, -1); + string testdir = run_dir + "/" + testname; + int status; + + argv[0] = argv0.c_str(); + argv[1] = "-r"; + argv[2] = testfile.c_str(); + argv[3] = firstdir.c_str(); + argv[4] = testname.c_str(); + argv[5] = 0; + + do_remove_recursive(testdir); + do_mkdir(testdir); + + { + change_current_working_dir(testdir); + pid_t child = process_spawn_redirected(NULL_DEVICE, + "tester.log", + "tester.log", + argv); + change_current_working_dir(run_dir); + process_wait(child, &status); + } + + // Set up a call to REPORTER with the appropriate arguments ... + lua_pushvalue(L, reporter); // t_r r tno tna r + lua_insert(L, -2); // t_r r tno r tna + lua_pushvalue(L, -3); // t_r r tno r tna tno + lua_insert(L, -2); // t_r r tno r tno tna + lua_pushinteger(L, status); // t_r r tno r tno tna st + + // ... call it ... + lua_call(L, 3, 1); + + // ... and if it returns true, delete testdir. + I(lua_isboolean(L, -1)); + if (lua_toboolean(L, -1)) + do_remove_recursive(testdir); + + // pop return value, leaving to_run reporter testno, as expected by + // lua_next. + lua_remove(L, -1); + } return 0; } @@ -831,18 +860,113 @@ int main(int argc, char **argv) { int retcode = 2; lua_State *st = 0; + try { - string testfile; - string firstdir; - bool needhelp = false; - for (int i = 1; i < argc; ++i) - if (string(argv[i]) == "--help" || string(argv[i]) == "-h") - needhelp = true; - if (argc > 1 && !needhelp) + vector tests_to_run; + bool want_help = false; + bool need_help = false; + bool debugging = false; + bool list_only = false; + bool run_one = false; + + option::concrete_option_set os; + os("help,h", "display help message", option::setter(want_help)) + ("debug,d", "don't erase per-test directories for successful tests", + option::setter(debugging)) + ("list,l", "list tests that would be run, but don't run them", + option::setter(list_only)) + ("run-one,r", "", // internal use only! + option::setter(run_one)) + ("--", "", option::setter(tests_to_run)); + + try { + os.from_command_line(argc, argv); + } + catch (option::option_error & e) + { + P(F("%s: %s\n") % argv[0] % e.what()); + need_help = true; + } + + if (tests_to_run.size() == 0) + { + P(F("%s: no test suite specified")); + need_help = true; + } + + if (run_one && (want_help || debugging || list_only + || tests_to_run.size() != 3)) + { + P(F("%s: incorrect self-invocation") % argv[0]); + need_help = true; + } + + if (want_help || need_help) + { + P(F("Usage: %s test-file testsuite [options] [tests]\n") % argv[0]); + P(F("Testsuite: a Lua script defining the test suite to run.\n")); + P(F("Options:\n%s\n") % os.get_usage_str()); + P(F("Tests may be specified as:\n" + " nothing - run all tests.\n" + " numbers - run the tests with those numbers\n" + " negative numbers count back from the end\n" + " ranges may be specified as A..B (inclusive)\n" + " regexes - run the tests whose names match (unanchored)\n")); + + return want_help ? 0 : 2; + } + + st = luaL_newstate(); + lua_atpanic (st, &panic_thrower); + luaL_openlibs(st); + add_functions(st); + + if (run_one) + { + // This is a self-invocation, requesting that we actually run a + // single named test. Contra the help above, the command line + // arguments are the absolute pathname of the testsuite definition, + // the original working directory, and the name of the test, in + // that order. No other options are valid in combination with -r. + // We have been invoked inside the directory where we should run + // the test. Stdout and stderr have been redirected to a per-test + // logfile. + + lua_pushstring(st, tests_to_run[1].c_str()); + lua_setglobal(st, "initial_dir"); + + source_dir = dirname(tests_to_run[0]); + + run_string(st, testlib_constant, "testlib.lua"); + run_file(st, tests_to_run[0].c_str()); + + Lua ll(st); + ll.func("run_one_test"); + ll.push_str(tests_to_run[2]); + ll.call(1,1) + .extract_int(retcode); + } + else + { firstdir = get_current_working_dir(); run_dir = firstdir + "/tester_dir"; + testfile = tests_to_run.front(); + + if (argv[0][0] == '/' +#ifdef WIN32 + || argv[0][0] != '\0' && argv[0][1] == ':' +#endif + ) + argv0 = argv[0]; + else + argv0 = firstdir + "/" + argv[0]; + + change_current_working_dir(dirname(testfile)); + source_dir = get_current_working_dir(); + testfile = source_dir + "/" + basename(testfile); + switch (get_path_status(run_dir)) { case path::directory: break; @@ -853,70 +977,47 @@ int main(int argc, char **argv) do_mkdir(run_dir); } - testfile = argv[1]; - change_current_working_dir(dirname(testfile)); - source_dir = get_current_working_dir(); - testfile = source_dir + "/" + basename(testfile); - change_current_working_dir(run_dir); - } - else - { - P(F("Usage: %s test-file [arguments]\n") % argv[0]); - P(F("\t-h print this message\n")); - P(F("\t-l print test names only; don't run them\n")); - P(F("\t-d don't clean the scratch directories\n")); - P(F("\tnum run a specific test\n")); - P(F("\tnum..num run tests in a range\n")); - P(F("\t if num is negative, count back from the end\n")); - P(F("\tregex run tests with matching names\n")); - return needhelp ? 0 : 1; - } - st = luaL_newstate(); - lua_atpanic (st, &panic_thrower); - luaL_openlibs(st); - add_functions(st); - - lua_pushstring(st, firstdir.c_str()); - lua_setglobal(st, "initial_dir"); - run_string(st, testlib_constant, "tester builtin functions"); - run_file(st, testfile.c_str()); + run_string(st, testlib_constant, "testlib.lua"); + lua_pushstring(st, firstdir.c_str()); + lua_setglobal(st, "initial_dir"); + run_file(st, testfile.c_str()); - // arrange for isolation between different test suites running in the - // same build directory. - { - lua_getglobal(st, "testdir"); - const char *testdir = lua_tostring(st, 1); - I(testdir); - string testdir_base = basename(testdir); - run_dir = run_dir + "/" + testdir_base; - string logfile = run_dir + ".log"; - switch (get_path_status(run_dir)) - { - case path::directory: break; - case path::file: - P(F("cannot create directory '%s': it is a file") % run_dir); - return 1; - case path::nonexistent: - do_mkdir(run_dir); - } + // arrange for isolation between different test suites running in + // the same build directory. + lua_getglobal(st, "testdir"); + const char *testdir = lua_tostring(st, 1); + I(testdir); + string testdir_base = basename(testdir); + run_dir = run_dir + "/" + testdir_base; + string logfile = run_dir + ".log"; + switch (get_path_status(run_dir)) + { + case path::directory: break; + case path::file: + P(F("cannot create directory '%s': it is a file") % run_dir); + return 1; + case path::nonexistent: + do_mkdir(run_dir); + } - lua_pushstring(st, logfile.c_str()); - lua_setglobal(st, "logfile"); - } - - Lua ll(st); - ll.func("run_tests"); - ll.push_table(); - for (int i = 2; i < argc; ++i) - { - ll.push_int(i-1); - ll.push_str(argv[i]); - ll.set_table(); + Lua ll(st); + ll.func("run_tests"); + ll.push_bool(debugging); + ll.push_bool(list_only); + ll.push_str(run_dir); + ll.push_str(logfile); + ll.push_table(); + for (int i = 2; i < argc; ++i) + { + ll.push_int(i-1); + ll.push_str(argv[i]); + ll.set_table(); + } + ll.call(5,1) + .extract_int(retcode); } - ll.call(1,1) - .extract_int(retcode); } catch (informative_failure & e) { @@ -944,6 +1045,137 @@ int main(int argc, char **argv) return retcode; } +// The functions below are used by option.cc, and cloned from ui.cc and +// simplestring_xform.cc, which we cannot use here. They do not cover +// several possibilities handled by the real versions. + +unsigned int +guess_terminal_width() +{ + unsigned int w = terminal_width(); + if (!w) + w = 80; // can't use constants:: here + return w; +} + +static void +split_into_lines(string const & in, vector & out) +{ + out.clear(); + string::size_type begin = 0; + string::size_type end = in.find_first_of("\r\n", begin); + while (end != string::npos && end >= begin) + { + out.push_back(in.substr(begin, end-begin)); + if (in.at(end) == '\r' + && in.size() > end+1 + && in.at(end+1) == '\n') + begin = end + 2; + else + begin = end + 1; + if (begin >= in.size()) + break; + end = in.find_first_of("\r\n", begin); + } + if (begin < in.size()) + out.push_back(in.substr(begin, in.size() - begin)); +} + +static vector split_into_words(string const & in) +{ + vector out; + + string::size_type begin = 0; + string::size_type end = in.find_first_of(" ", begin); + + while (end != string::npos && end >= begin) + { + out.push_back(in.substr(begin, end-begin)); + begin = end + 1; + if (begin >= in.size()) + break; + end = in.find_first_of(" ", begin); + } + if (begin < in.size()) + out.push_back(in.substr(begin, in.size() - begin)); + + return out; +} + +// See description for format_text below for more details. +static string +format_paragraph(string const & text, size_t const col, size_t curcol) +{ + I(text.find('\n') == string::npos); + + string formatted; + if (curcol < col) + { + formatted = string(col - curcol, ' '); + curcol = col; + } + + const size_t maxcol = guess_terminal_width(); + + vector< string > words = split_into_words(text); + for (vector< string >::const_iterator iter = words.begin(); + iter != words.end(); iter++) + { + string const & word = *iter; + + if (iter != words.begin() && + curcol + word.size() + 1 > maxcol) + { + formatted += '\n' + string(col, ' '); + curcol = col; + } + else if (iter != words.begin()) + { + formatted += ' '; + curcol++; + } + + formatted += word; + curcol += word.size(); + } + + return formatted; +} + +// Reformats the given text so that it fits in the current screen with no +// wrapping. +// +// The input text is a series of words and sentences. Paragraphs may be +// separated with a '\n' character, which is taken into account to do the +// proper formatting. The text should not finish in '\n'. +// +// 'col' specifies the column where the text will start and 'curcol' +// specifies the current position of the cursor. +string +format_text(string const & text, size_t const col, size_t curcol) +{ + I(curcol <= col); + + string formatted; + + vector< string > lines; + split_into_lines(text, lines); + for (vector< string >::const_iterator iter = lines.begin(); + iter != lines.end(); iter++) + { + string const & line = *iter; + + formatted += format_paragraph(line, col, curcol); + if (iter + 1 != lines.end()) + formatted += "\n\n"; + curcol = 0; + } + + return formatted; +} + + + // Local Variables: // mode: C++ // fill-column: 76 ============================================================ --- testlib.lua 8c72b168db4237136c272bded45474b1ca43da38 +++ testlib.lua df3aee450ca03e7454704cde586448214cb6e2c5 @@ -11,8 +11,6 @@ logfile = nil -- combined logfile; tester.cc will reset this to a filename, which is -- then opened in run_tests logfile = nil --- logfiles of failed tests; append these to the main logfile -failed_testlogs = {} -- This is for redirected output from local implementations -- of shellutils type stuff (ie, grep). @@ -54,12 +52,6 @@ end return 0 end -function P(...) - io.write(unpack(arg)) - io.flush() - logfile:write(unpack(arg)) -end - function L(...) test.log:write(unpack(arg)) test.log:flush() @@ -400,7 +392,8 @@ function runcmd(cmd, prefix, bgnd) result = {pcall(execute, unpack(cmd))} end else - err("runcmd called with bad command table") + err("runcmd called with bad command table " .. + "(first entry is a " .. type(cmd[1]) ..")") end if local_redir then @@ -815,11 +808,16 @@ end end end -function run_tests(args) +function run_tests(debugging, list_only, run_dir, logname, args) local torun = {} local run_all = true - local list_only = false + local function P(...) + io.write(unpack(arg)) + io.flush() + logfile:write(unpack(arg)) + end + -- NLS nuisances. for _,name in pairs({ "LANG", "LANGUAGE", @@ -842,8 +840,8 @@ function run_tests(args) -- no test suite should touch the user's ssh agent unset_env("SSH_AUTH_SOCK") - -- tester.cc has set 'logfile' to an appropriate file name - logfile = io.open(logfile, "w") + logfile = io.open(logname, "w") + chdir(run_dir); do local s = prepare_to_enumerate_tests() @@ -853,6 +851,7 @@ function run_tests(args) end end + -- testdir is set by the testsuite definition -- any directory in testdir with a __driver__.lua inside is a test case local tests = {} for _,candidate in ipairs(read_directory(testdir)) do @@ -874,45 +873,50 @@ function run_tests(args) if r < 1 then r = table.getn(tests) + r + 1 end if l > r then l,r = r,l end for j = l,r do - torun[j]=j + torun[j] = tests[j] end run_all = false elseif string.find(a, "^-?%d+$") then r = a + 0 if r < 1 then r = table.getn(tests) + r + 1 end - torun[r] = r + torun[r] = tests[r] run_all = false - elseif a == "-d" then - debugging = true - elseif a == "-l" then - list_only = true else -- pattern + run_all = false local matched = false for i,t in pairs(tests) do if regex.search(a, t) then - torun[i] = i + torun[i] = t matched = true end end - if matched then - run_all = false - else + if not matched then print(string.format("Warning: pattern '%s' does not match any tests.", a)) end end end - if not list_only then - logfile:write("Running on ", get_ostype(), "\n\n") - local s = prepare_to_run_tests() - if s ~= 0 then - P("Test suite preparation failed.\n") - return s + if run_all then torun = tests end + + if list_only then + for i,t in pairs(torun) do + if i < 10 then P(" ") end + if i < 100 then P(" ") end + P(i .. " " .. t .. "\n") end - P("Running tests...\n") + logfile:close() + return 0 end + logfile:write("Running on ", get_ostype(), "\n\n") + local s = prepare_to_run_tests() + if s ~= 0 then + P("Test suite preparation failed.\n") + return s + end + P("Running tests...\n") + local counts = {} counts.success = 0 counts.skip = 0 @@ -922,169 +926,69 @@ function run_tests(args) counts.total = 0 counts.of_interest = 0 local of_interest = {} + local failed_testlogs = {} - local function runtest(i, tname) - save_env() - local env = {} - local genv = getfenv(0) - for x,y in pairs(genv) do - env[x] = y - -- we want changes to globals in a test to be visible to - -- already-defined functions - if type(y) == "function" then - pcall(setfenv, y, env) - end - end - env.tests = nil -- don't let them mess with this - - test.bgid = 0 - test.name = tname - test.wanted_fail = false - test.partial_skip = false - local shortname = nil - test.root, shortname = go_to_test_dir(tname) - if shortname == nil or test.root == nil then - P("ERROR: could not enter scratch dir for test '", tname, "'\n") - error("") - end - test.errfile = "" - test.errline = -1 - test.bglist = {} - - local test_header = "" - if i < 100 then test_header = test_header .. " " end - if i < 10 then test_header = test_header .. " " end - test_header = test_header .. i .. " " .. shortname .. " " - local spacelen = 45 - string.len(shortname) - local spaces = string.rep(" ", 50) - if spacelen > 0 then - test_header = test_header .. string.sub(spaces, 1, spacelen) - end - P(test_header) - - local tlog = test.root .. "/tester.log" - test.log = io.open(tlog, "w") - L("Test number ", i, ", ", shortname, "\n") - - local driverfile = testdir .. "/" .. test.name .. "/__driver__.lua" - local driver, e = loadfile(driverfile) - local r - if driver == nil then - r = false - e = "Could not load driver file " .. driverfile .. " .\n" .. e - else - setfenv(driver, env) - local oldmask = posix_umask(0) - posix_umask(oldmask) - r,e = xpcall(driver, debug.traceback) - local errline = test.errline - for i,b in pairs(test.bglist) do - local a,x = pcall(function () b:finish(0) end) - if r and not a then - r = a - e = x - elseif not a then - L("Error cleaning up background processes: ", tostring(b.locstr), "\n") - end - end - if type(env.cleanup) == "function" then - local a,b = pcall(env.cleanup) - if r and not a then - r = a - e = b - end - end - test.errline = errline - posix_umask(oldmask) - end - - -- set our functions back to the proper environment - local genv = getfenv(0) - for x,y in pairs(genv) do - if type(y) == "function" then - pcall(setfenv, y, genv) - end - end - - if r then - if test.wanted_fail then - P("unexpected success\n") - test.log:close() - leave_test_dir() - counts.noxfail = counts.noxfail + 1 - counts.of_interest = counts.of_interest + 1 - table.insert(of_interest, test_header .. "unexpected success") - else - if test.partial_skip then - P("partial skip\n") - else - P("ok\n") - end - test.log:close() - if not debugging then clean_test_dir(tname) end - counts.success = counts.success + 1 - end - else - if test.errline == nil then test.errline = -1 end - if type(e) ~= "table" then - local tbl = {e = e, bt = {"no backtrace; type(err) = "..type(e)}} - e = tbl - end - if e.e == true then - P(string.format("skipped (line %i)\n", test.errline)) - test.log:close() - if not debugging then clean_test_dir(tname) end - counts.skip = counts.skip + 1 - elseif e.e == false then - P(string.format("expected failure (line %i)\n", test.errline)) - test.log:close() - leave_test_dir() - counts.xfail = counts.xfail + 1 - else - result = string.format("FAIL (line %i)", test.errline) - P(result, "\n") - log_error(e) - table.insert(failed_testlogs, tlog) - test.log:close() - leave_test_dir() - counts.fail = counts.fail + 1 - counts.of_interest = counts.of_interest + 1 - table.insert(of_interest, test_header .. result) - end - end - counts.total = counts.total + 1 - restore_env() + -- callback closure passed to run_tests_in_children + function report_one_test(tno, tname, status) + local tdir = run_dir .. "/" .. tname + local test_header = string.format("%3d %-45s ", tno, tname) + local what + local can_delete + -- the child should always exit successfully, just to avoid + -- headaches. if we get any other code we report it as a failure. + if status ~= 0 then + if status < 0 then + what = string.format("FAIL (signal %d)", -status) + else + what = string.format("FAIL (exit %d)", status) + end + counts.fail = counts.fail + 1 + table.insert(of_interest, test_header .. what) + table.insert(failed_testlogs, tdir .. "/tester.log") + can_delete = false + else + local wfile = io.open(tdir .. "/STATUS", "r") + what = wfile:read() + wfile:close() + if what == "unexpected success" then + counts.noxfail = counts.noxfail + 1 + counts.of_interest = counts.of_interest + 1 + table.insert(of_interest, test_header .. "unexpected success") + can_delete = false + elseif what == "partial skip" or what == "ok" then + counts.success = counts.success + 1 + can_delete = true + elseif string.find(what, "skipped ") == 1 then + counts.skip = counts.skip + 1 + can_delete = true + elseif string.find(what, "expected failure ") == 1 then + counts.xfail = counts.xfail + 1 + can_delete = false + elseif string.find(what, "FAIL ") == 1 then + counts.fail = counts.fail + 1 + table.insert(of_interest, test_header .. what) + table.insert(failed_testlogs, tdir .. "/tester.log") + can_delete = false + else + counts.fail = counts.fail + 1 + what = "FAIL (gobbledygook: " .. what .. ")" + table.insert(of_interest, test_header .. what) + table.insert(failed_testlogs, tdir .. "/tester.log") + can_delete = false + end + end + counts.total = counts.total + 1 + P(string.format("%s%s\n", test_header, what)) + if debugging then + return false + else + unlogged_remove(tdir .. "/STATUS") + return can_delete + end end - if run_all then - for i,t in pairs(tests) do - if list_only then - if i < 10 then P(" ") end - if i < 100 then P(" ") end - P(i .. " " .. t .. "\n") - else - runtest(i, t) - end - end - else - for i,t in pairs(tests) do - if torun[i] == i then - if list_only then - if i < 10 then P(" ") end - if i < 100 then P(" ") end - P(i .. " " .. t .. "\n") - else - runtest(i, t) - end - end - end - end - - if list_only then - logfile:close() - return 0 - end - + run_tests_in_children(torun, report_one_test) + if counts.of_interest ~= 0 and (counts.total / counts.of_interest) > 4 then P("\nInteresting tests:\n") for i,x in ipairs(of_interest) do @@ -1092,12 +996,6 @@ function run_tests(args) end end P("\n") - P(string.format("Of %i tests run:\n", counts.total)) - P(string.format("\t%i succeeded\n", counts.success)) - P(string.format("\t%i failed\n", counts.fail)) - P(string.format("\t%i had expected failures\n", counts.xfail)) - P(string.format("\t%i succeeded unexpectedly\n", counts.noxfail)) - P(string.format("\t%i were skipped\n", counts.skip)) for i,log in pairs(failed_testlogs) do local tlog = io.open(log, "r") @@ -1108,11 +1006,101 @@ function run_tests(args) logfile:write(dat) end end - logfile:close() + -- Write out this summary in one go so that it does not get interrupted + -- by concurrent test suites' summaries. + P(string.format("Of %i tests run:\n".. + "\t%i succeeded\n".. + "\t%i failed\n".. + "\t%i had expected failures\n".. + "\t%i succeeded unexpectedly\n".. + "\t%i were skipped\n", + counts.total, counts.success, counts.fail, + counts.xfail, counts.noxfail, counts.skip)) + + logfile:close() if counts.success + counts.skip + counts.xfail == counts.total then return 0 else return 1 end end + +function run_one_test(tname) + test.bgid = 0 + test.name = tname + test.wanted_fail = false + test.partial_skip = false + test.root = chdir(".") + test.errfile = "" + test.errline = -1 + test.bglist = {} + test.log = io.output() + + L("Test ", test.name, "\n") + + local driverfile = testdir .. "/" .. test.name .. "/__driver__.lua" + local driver, e = loadfile(driverfile) + local r + if driver == nil then + r = false + e = "Could not load driver file " .. driverfile .. ".\n" .. e + else + local oldmask = posix_umask(0) + posix_umask(oldmask) + r,e = xpcall(driver, debug.traceback) + local errline = test.errline + for i,b in pairs(test.bglist) do + local a,x = pcall(function () b:finish(0) end) + if r and not a then + r = a + e = x + elseif not a then + L("Error cleaning up background processes: ", + tostring(b.locstr), "\n") + end + end + if type(cleanup) == "function" then + local a,b = pcall(cleanup) + if r and not a then + r = a + e = b + end + end + test.errline = errline + posix_umask(oldmask) + end + + -- record the short status where report_one_test can find it + local s = io.open(test.root .. "/STATUS", "w") + + if r then + if test.wanted_fail then + s:write("unexpected success\n") + else + if test.partial_skip then + s:write("partial skip\n") + else + s:write("ok\n") + end + end + else + if test.errline == nil then test.errline = -1 end + if type(e) ~= "table" then + local tbl = {e = e, bt = {"no backtrace; type(err) = "..type(e)}} + e = tbl + end + if e.e == true then + s:write(string.format("skipped (line %i)\n", test.errline)) + elseif e.e == false then + s:write(string.format("expected failure (line %i)\n", + test.errline)) + else + s:write(string.format("FAIL (line %i)\n", test.errline)) + log_error(e) + end + end + test.log:close() + s:close() + return 0 +end ============================================================ --- testsuite.lua 9ca3325fc516d2e6235f800c444e1152562d8eae +++ testsuite.lua 0ae9af3c12cea3f293bd302fde0c54c4bfa45a38 @@ -1,6 +1,14 @@ #!./tester +monotone_path = nil + function safe_mtn(...) + if monotone_path == nil then + monotone_path = os.getenv("mtn") + if monotone_path == nil then + err("'mtn' environment variable not set") + end + end return {monotone_path, "--norc", "--root=" .. test.root, "--confdir="..test.root, unpack(arg)} end