From 8ee3ca4bb96f61ed5fc12da1d751a29fec6f7979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Sat, 2 May 2015 10:52:37 +0100 Subject: [PATCH 1/5] tests: fix races in and standardize the tail tests * tests/tail-2/F-vs-missing.sh: Use standard "fastpoll" options (-s.1 --max-unchanged-stats=1) to speedup the non-inotify case. Add the non-inotify case to the test. `wait` on the background tail process to terminate which should avoid the need for the non standard `retry_delay_ cleanup ...` on NFS. * tests/tail-2/F-vs-rename.sh: Remove 'out' at the start of the loop, to avoid a race in checking its contents. Also ensure 'a' & 'b' files are present before the tail process starts. Use the standard "fastpoll" options as above. * tests/tail-2/f-vs-rename.sh: Likewise. * tests/tail-2/append-only.sh: Use more standard variable names. * tests/tail-2/flush-initial.sh: Use "fastpoll" options for non-inotify platforms. Also `wait` on the background tail to avoid stray processes and file cleanup issues on NFS. * tests/tail-2/inotify-hash-abuse.sh: Always run non-inotify case. Use "fastpoll" options. Use a more standard retry_delay_ instead of a hardcoded sleep loop. Add a `wait` on the background tail. * tests/tail-2/inotify-hash-abuse2.sh: Likewise. * tests/tail-2/inotify-rotate-resources.sh: Wait just on the specific tail $pid needed. * tests/tail-2/inotify-rotate.sh: Use "fastpoll" options. * tests/tail-2/pid.sh: Use standard variable names. Add a `wait` on the background tails. * tests/tail-2/pipe-f2.sh: Likewise. * tests/tail-2/tail-n0f.sh: Likewise. * tests/tail-2/retry.sh: Use "fastpoll" options. * tests/tail-2/symlink.sh: Likewise. * tests/tail-2/wait.sh: Likewise. Speedup by using sub second parameters to timeout(1). Improve the part ensuring that -F never follows a renamed file. * tests/tail-2/infloop-1.sh: Remove invalid test. tail(1) was not being passed the --pid=$yes_pid option, retry_delay_ wasn't used to avoid races, and yes could write huge files before being killed. * tests/local.mk: Remove the invalid test reference. * tests/tail-2/assert-2.sh: Rewrite using retry_delay_(). Since no longer hardcoding large delays, remove the VERY_EXPENSIVE tag. * tests/tail-2/assert.sh: Likewise. --- tests/local.mk | 1 - tests/tail-2/F-vs-missing.sh | 48 ++++++++++------------ tests/tail-2/F-vs-rename.sh | 15 ++++--- tests/tail-2/append-only.sh | 7 ++-- tests/tail-2/assert-2.sh | 47 ++++++++++++---------- tests/tail-2/assert.sh | 68 +++++++++++++++++--------------- tests/tail-2/f-vs-rename.sh | 14 ++++--- tests/tail-2/flush-initial.sh | 10 +++-- tests/tail-2/infloop-1.sh | 46 --------------------- tests/tail-2/inotify-hash-abuse.sh | 51 +++++++++++++----------- tests/tail-2/inotify-hash-abuse2.sh | 28 +++++++------ tests/tail-2/inotify-rotate-resources.sh | 14 ++++--- tests/tail-2/inotify-rotate.sh | 6 ++- tests/tail-2/pid.sh | 14 +++---- tests/tail-2/pipe-f2.sh | 7 +++- tests/tail-2/retry.sh | 41 +++++++++++-------- tests/tail-2/symlink.sh | 26 +++++++----- tests/tail-2/tail-n0f.sh | 6 +-- tests/tail-2/wait.sh | 27 +++++++------ 19 files changed, 244 insertions(+), 232 deletions(-) delete mode 100755 tests/tail-2/infloop-1.sh diff --git a/tests/local.mk b/tests/local.mk index 1be31ad..0d4f9df 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -197,7 +197,6 @@ all_tests = \ tests/rm/i-1.sh \ tests/rm/i-never.sh \ tests/rm/i-no-r.sh \ - tests/tail-2/infloop-1.sh \ tests/rm/ignorable.sh \ tests/rm/inaccessible.sh \ tests/rm/interactive-always.sh \ diff --git a/tests/tail-2/F-vs-missing.sh b/tests/tail-2/F-vs-missing.sh index 5d2cc2f..6622f7e 100755 --- a/tests/tail-2/F-vs-missing.sh +++ b/tests/tail-2/F-vs-missing.sh @@ -21,10 +21,6 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -debug='---disable-inotify' -debug= -tail $debug -F -s.1 missing/file > out 2>&1 & pid=$! - check_tail_output() { local delay="$1" @@ -32,33 +28,31 @@ check_tail_output() { sleep $delay; return 1; } } -# Wait up to 12.7s for tail to start with diagnostic: -# tail: cannot open 'missing/file' for reading: No such file or directory -tail_re='cannot open' retry_delay_ check_tail_output .1 7 || fail=1 +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' -mkdir missing || fail=1 -(cd missing && echo x > file) +for mode in '' '---disable-inotify'; do + rm -rf out missing -# Wait up to 12.7s for this to appear in the output: -# "tail: '...' has appeared; following end of new file" -tail_re='has appeared' retry_delay_ check_tail_output .1 7 || - { echo "$0: file: unexpected delay?"; cat out; fail=1; } + tail $mode -F $fastpoll missing/file > out 2>&1 & pid=$! -kill -HUP $pid + # Wait up to 12.7s for tail to start with diagnostic: + # tail: cannot open 'missing/file' for reading: No such file or directory + tail_re='cannot open' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } -cleanup() -{ - local delay="$1" - rm -rf missing || - { sleep $delay; return 1; } -} + mkdir missing || framework_failure_ + (cd missing && echo x > file) || framework_failure_ + + # Wait up to 12.7s for this to appear in the output: + # "tail: '...' has appeared; following end of new file" + tail_re='has appeared' retry_delay_ check_tail_output .1 7 || + { echo "$0: file: unexpected delay?"; cat out; fail=1; } + + kill $pid + + wait $pid +done -# Try repeatedly to remove the temporary directory. -# This is normally unnecessary, because the containing directory will -# be removed by code from init.sh. However, when this particular test -# is run on an NFS-mounted volume, sometimes init.sh's cleanup code -# fails because the directory is not yet really empty, perhaps because -# the tail process (reading missing/file) is not yet killed. -retry_delay_ cleanup .1 6 Exit $fail diff --git a/tests/tail-2/F-vs-rename.sh b/tests/tail-2/F-vs-rename.sh index f95b71a..e087ec2 100755 --- a/tests/tail-2/F-vs-rename.sh +++ b/tests/tail-2/F-vs-rename.sh @@ -28,20 +28,25 @@ check_tail_output() { sleep $delay; return 1; } } -touch a b || framework_failure_ +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' for mode in '' '---disable-inotify'; do - tail $mode -F -s.1 a b > out 2>&1 & pid=$! + rm -f a b out + touch a b || framework_failure_ + + tail $mode -F $fastpoll a b > out 2>&1 & pid=$! # Wait up to 12.7s for tail to start. echo x > a - tail_re='^x$' retry_delay_ check_tail_output .1 7 || fail=1 + tail_re='^x$' retry_delay_ check_tail_output .1 7 || { cat out; fail=1; } - mv a b || fail=1 + mv a b || framework_failure_ # Wait 12.7s for this diagnostic: # tail: 'a' has become inaccessible: No such file or directory - tail_re='inaccessible' retry_delay_ check_tail_output .1 7 || fail=1 + tail_re='inaccessible' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } echo x > a # Wait up to 12.7s for this to appear in the output: diff --git a/tests/tail-2/append-only.sh b/tests/tail-2/append-only.sh index 8baba7e..9da10ae 100755 --- a/tests/tail-2/append-only.sh +++ b/tests/tail-2/append-only.sh @@ -32,10 +32,9 @@ if test $chattr_a_works = 0; then fi -for inotify in ---disable-inotify ''; do - sleep 1 & - pid=$! - tail --pid=$pid -f $inotify f || fail=1 +for mode in '' '---disable-inotify'; do + sleep 1 & pid=$! + tail --pid=$pid -f $mode f || fail=1 done chattr -a f 2>/dev/null diff --git a/tests/tail-2/assert-2.sh b/tests/tail-2/assert-2.sh index 2d4671b..aaf8785 100755 --- a/tests/tail-2/assert-2.sh +++ b/tests/tail-2/assert-2.sh @@ -1,5 +1,5 @@ #!/bin/sh -# This variant of 'assert' would get a UMR reliably in 2.0.9. +# This variant of 'assert' would get a Uninit Mem Read reliably in 2.0.9. # Due to a race condition in the test, the 'assert' script would get # the UMR on Solaris only some of the time, and not at all on Linux/GNU. @@ -21,29 +21,36 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -# Not "expensive" per se, but sleeping for so long is annoying. -very_expensive_ +check_tail_output() +{ + local delay="$1" + grep "$tail_re" out || + { sleep $delay; return 1; } +} -ok='ok ok ok' +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' -touch a -tail --follow=name a foo > err 2>&1 & -tail_pid=$! -# Arrange for the tail process to die after 12 seconds. -(sleep 12; kill $tail_pid) & -echo $ok > f -echo sleeping for 7 seconds... -sleep 7 -mv f foo -# echo waiting.... -wait +for mode in '' '---disable-inotify'; do + rm -f a foo out + touch a || framework_failure_ -case "$(cat err)" in - *$ok) ;; - *) fail=1;; -esac + tail $mode --follow=name $fastpoll a foo > out 2>&1 & pid=$! -test $fail = 1 && cat err + # Wait up to 12.7s for tail to start. + echo x > a || framework_failure_ + tail_re='^x$' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + # Wait up to 12.7s for tail to notice new foo file + ok='ok ok ok' + echo "$ok" > foo || framework_failure_ + tail_re="^$ok$" retry_delay_ check_tail_output .1 7 || + { echo "$0: foo: unexpected delay?"; cat out; fail=1; } + + kill $pid + wait $pid +done Exit $fail diff --git a/tests/tail-2/assert.sh b/tests/tail-2/assert.sh index faf7e32..906e364 100755 --- a/tests/tail-2/assert.sh +++ b/tests/tail-2/assert.sh @@ -26,36 +26,42 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -# Not "expensive" per se, but sleeping for so long is annoying. -very_expensive_ - -ok='ok ok ok' - -touch a foo -tail --follow=name a foo > err 2>&1 & -tail_pid=$! -# Arrange for the tail process to die after 12 seconds. -(sleep 12; kill $tail_pid) & - -echo sleeping for 7 seconds... - -# Give the backgrounded 'tail' a chance to start before removing foo. -# Otherwise, without --retry, tail wouldn't try to open 'foo' again. -sleep 1 - -rm -f foo -sleep 6 -echo $ok > f -mv f foo - -# echo waiting.... -wait - -case "$(cat err)" in - *$ok) ;; - *) fail=1;; -esac - -test $fail = 1 && cat err +check_tail_output() +{ + local delay="$1" + grep "$tail_re" out || + { sleep $delay; return 1; } +} + +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + + +for mode in '' '---disable-inotify'; do + rm -f a foo out + touch a foo || framework_failure_ + + tail $mode --follow=name $fastpoll a foo > out 2>&1 & pid=$! + + # Wait up to 12.7s for tail to start. + echo x > a || framework_failure_ + tail_re='^x$' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + # Wait 12.7s for this diagnostic: + # tail: foo: No such file or directory + rm foo || framework_failure_ + tail_re='No such file' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + # Wait up to 12.7s for tail to notice new foo file + ok='ok ok ok' + echo "$ok" > foo || framework_failure_ + tail_re="^$ok$" retry_delay_ check_tail_output .1 7 || + { echo "$0: foo: unexpected delay?"; cat out; fail=1; } + + kill $pid + wait $pid +done Exit $fail diff --git a/tests/tail-2/f-vs-rename.sh b/tests/tail-2/f-vs-rename.sh index 4bff41c..bfd0949 100755 --- a/tests/tail-2/f-vs-rename.sh +++ b/tests/tail-2/f-vs-rename.sh @@ -28,20 +28,24 @@ check_tail_output() { sleep $delay; return 1; } } -touch a || framework_failure_ +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' for mode in '' '---disable-inotify'; do - tail $mode -f -s.1 a > out 2>&1 & pid=$! + rm -f a out + touch a || framework_failure_ + + tail $mode $fastpoll -f a > out 2>&1 & pid=$! # Wait up to 12.7s for tail to start. echo x > a - tail_re='^x$' retry_delay_ check_tail_output .1 7 || fail=1 + tail_re='^x$' retry_delay_ check_tail_output .1 7 || { cat out; fail=1; } - mv a b || fail=1 + mv a b || framework_failure_ echo y >> b # Wait up to 12.7s for "y" to appear in the output: - tail_re='^y$' retry_delay_ check_tail_output .1 7 || fail=1 + tail_re='^y$' retry_delay_ check_tail_output .1 7 || { cat out; fail=1; } kill $pid diff --git a/tests/tail-2/flush-initial.sh b/tests/tail-2/flush-initial.sh index 1c8d6b0..aaeb878 100755 --- a/tests/tail-2/flush-initial.sh +++ b/tests/tail-2/flush-initial.sh @@ -19,11 +19,13 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -echo line > in || fail=1 +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + +echo line > in || framework_failure_ # Output should be buffered since we're writing to file # so we're depending on the flush to write out -tail -f in > out & -tail_pid=$! +tail $fastpoll -f in > out & tail_pid=$! # Wait for 3.1s for the file to be flushed. tail_flush() @@ -37,4 +39,6 @@ retry_delay_ tail_flush .1 5 || fail=1 kill $tail_pid +wait $tail_pid + Exit $fail diff --git a/tests/tail-2/infloop-1.sh b/tests/tail-2/infloop-1.sh deleted file mode 100755 index 61f9719..0000000 --- a/tests/tail-2/infloop-1.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# This test would fail with tail from pre-1.22i textutils. - -# Copyright (C) 1999-2015 Free Software Foundation, Inc. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src -print_ver_ tail - -yes > t & -yes_pid=$! -while :; do - test -s t \ - && break - sleep .1 -done -tail -n 1 t & -tail_pid=$! -kill $yes_pid - -# This test is racy, and can fail under unusual circumstances. -# On a very busy system, tail will fail to notice that $yes_pid is gone. -# Then the following kill will succeed and cause this test to fail. - -# Wait for up to 3 seconds for tail to detect the death of $yes_pid. -for i in $(seq 30); do - kill -0 $tail_pid || break - echo sleep 0.1s - sleep .1 -done - -kill $tail_pid && fail=1 || : - -Exit $fail diff --git a/tests/tail-2/inotify-hash-abuse.sh b/tests/tail-2/inotify-hash-abuse.sh index b819b97..d8c14c1 100755 --- a/tests/tail-2/inotify-hash-abuse.sh +++ b/tests/tail-2/inotify-hash-abuse.sh @@ -23,10 +23,6 @@ print_ver_ tail n=9 seq $n | xargs touch || framework_failure_ -debug='---disable-inotify' -debug= -tail $debug -s.1 -qF $(seq $n) > out 2>&1 & pid=$! - check_tail_output() { local delay="$1" @@ -34,30 +30,39 @@ check_tail_output() { sleep $delay; return 1; } } -# Wait up to 12.7s for tail to start -echo x > $n -tail_re='^x$' retry_delay_ check_tail_output .1 7 || fail=1 +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' -mv 1 f || fail=1 +for mode in '' '---disable-inotify'; do + rm -f out -# Wait 12.7s for this diagnostic: -# tail: '1' has become inaccessible: No such file or directory -tail_re='inaccessible' retry_delay_ check_tail_output .1 7 || fail=1 + tail $mode $fastpoll -qF $(seq $n) > out 2>&1 & pid=$! -# Trigger the bug. Before the fix, this would provoke the abort. -echo a > 1 || fail=1 + # Wait up to 12.7s for tail to start + echo x > $n + tail_re='^x$' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } -# Wait up to 2s for the buggy tail to die, -# or for the "tail: '1' has appeared; following end of new file" output -for i in $(seq 10); do - kill -0 $pid || break - grep 'has appeared;' out > /dev/null && break - sleep .2 -done + mv 1 f || framework_failure_ + + # Wait 12.7s for this diagnostic: + # tail: '1' has become inaccessible: No such file or directory + tail_re='inaccessible' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + # Trigger the bug. Before the fix, this would provoke the abort. + echo a > 1 || framework_failure_ -# Kill the working tail, or fail if it has already aborted -kill $pid || fail=1 + # Wait up to 6.3s for the "tail: '1' has appeared; ..." message + # (or for the buggy tail to die) + tail_re='has appeared' retry_delay_ check_tail_output .1 6 || + { cat out; fail=1; } + + # Kill the working tail, or fail if it has already aborted + kill $pid || fail=1 + + wait $pid +done -cat out Exit $fail diff --git a/tests/tail-2/inotify-hash-abuse2.sh b/tests/tail-2/inotify-hash-abuse2.sh index 241f26e..b054237 100755 --- a/tests/tail-2/inotify-hash-abuse2.sh +++ b/tests/tail-2/inotify-hash-abuse2.sh @@ -20,20 +20,24 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -touch f || framework_failure_ +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' -debug='---disable-inotify -s .001' -debug= -tail $debug -F f & pid=$! -cleanup_() { kill $pid; } +for mode in '' '---disable-inotify'; do + touch f || framework_failure_ -for i in $(seq 200); do - kill -0 $pid || break; - mv f g - touch f -done + tail $mode $fastpoll -F f & pid=$! + + for i in $(seq 200); do + kill -0 $pid || break; + mv f g + touch f + done -# Kill the working tail, or fail if it has already aborted -kill $pid || fail=1 + # Kill the working tail, or fail if it has already aborted + kill $pid || fail=1 + + wait $pid +done Exit $fail diff --git a/tests/tail-2/inotify-rotate-resources.sh b/tests/tail-2/inotify-rotate-resources.sh index a43f176..b65ceb7 100755 --- a/tests/tail-2/inotify-rotate-resources.sh +++ b/tests/tail-2/inotify-rotate-resources.sh @@ -34,10 +34,10 @@ check_tail_output() # Wait up to 25.5 seconds for grep REGEXP 'out' to succeed. grep_timeout() { tail_re="$1" retry_delay_ check_tail_output .1 8; } -strace_output() +check_strace() { local delay="$1" - test -s strace.out || + grep "$strace_re" strace.out > /dev/null || { sleep $delay; return 1; } } @@ -48,13 +48,15 @@ cleanup_fail() fail=1 } +fastpoll='-s.1 --max-unchanged-stats=1' + # Normally less than a second is required here, but with heavy load # and a lot of disk activity, even 20 seconds per grep_timeout is insufficient, # which leads to this timeout (used as a safety net for cleanup) # killing tail before processing is completed. touch k || framework_failure_ -timeout 180 strace -e inotify_rm_watch -o strace.out tail -F k >> out 2>&1 & -pid=$! +timeout 180 strace -e inotify_rm_watch -o strace.out \ + tail -F $fastpoll k >> out 2>&1 & pid=$! reverted_to_polling_=0 for i in $(seq 2); do @@ -82,7 +84,7 @@ for i in $(seq 2); do # may not be supported in future, instead being auto scaled to RAM # like the Linux epoll resources were. if test "$i" -gt 1; then - retry_delay_ strace_output .1 8 || + strace_re='inotify_rm_watch' retry_delay_ check_strace .1 8 || { cleanup_fail 'failed to find inotify_rm_watch syscall'; break; } fi @@ -91,6 +93,6 @@ done test "$reverted_to_polling_" = 1 && skip_ 'inotify resources already depleted' kill $pid -wait +wait $pid Exit $fail diff --git a/tests/tail-2/inotify-rotate.sh b/tests/tail-2/inotify-rotate.sh index 64724f9..be1f07e 100755 --- a/tests/tail-2/inotify-rotate.sh +++ b/tests/tail-2/inotify-rotate.sh @@ -40,6 +40,9 @@ cleanup_fail() fail=1 } +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + # Perform at least this many iterations, because on multi-core systems # the offending sequence of events can be surprisingly uncommon. # See: http://lists.gnu.org/archive/html/bug-coreutils/2009-11/msg00213.html @@ -51,8 +54,7 @@ for i in $(seq 50); do # and a lot of disk activity, even 20 seconds is insufficient, which # leads to this timeout killing tail before the "ok" is written below. >k && >x || framework_failure_ failed to initialize files - timeout 60 tail -s.1 --max-unchanged-stats=1 -F k > out 2>&1 & - pid=$! + timeout 60 tail $fastpoll -F k > out 2>&1 & pid=$! echo 'tailed' > k; # wait for 'tailed' to appear in out diff --git a/tests/tail-2/pid.sh b/tests/tail-2/pid.sh index e6b75a5..7a49867 100755 --- a/tests/tail-2/pid.sh +++ b/tests/tail-2/pid.sh @@ -23,28 +23,28 @@ getlimits_ touch empty here || framework_failure_ -for inotify in ---disable-inotify ''; do +for mode in '' '---disable-inotify'; do # Use tail itself to create a background process to monitor, # which will auto exit when "here" is removed. - tail -f $inotify here & - bg_pid=$! + tail -f $mode here & pid=$! # Ensure that tail --pid=PID does not exit when PID is alive. - timeout 1 tail -f -s.1 --pid=$bg_pid $inotify here + timeout 1 tail -f -s.1 --pid=$pid $mode here test $? = 124 || fail=1 # Cleanup background process - kill $bg_pid + kill $pid + wait $pid # Ensure that tail --pid=PID exits with success status when PID is dead. # Use an unlikely-to-be-live PID - timeout 10 tail -f -s.1 --pid=$PID_T_MAX $inotify empty + timeout 10 tail -f -s.1 --pid=$PID_T_MAX $mode empty ret=$? test $ret = 124 && skip_ "pid $PID_T_MAX present or tail too slow" test $ret = 0 || fail=1 # Ensure tail doesn't wait for data when PID is dead - timeout 10 tail -f -s10 --pid=$PID_T_MAX $inotify empty + timeout 10 tail -f -s10 --pid=$PID_T_MAX $mode empty test $? = 124 && fail=1 done diff --git a/tests/tail-2/pipe-f2.sh b/tests/tail-2/pipe-f2.sh index a40d552..4d5fe51 100755 --- a/tests/tail-2/pipe-f2.sh +++ b/tests/tail-2/pipe-f2.sh @@ -24,7 +24,10 @@ mkfifo_or_skip_ fifo echo 1 > fifo & echo 1 > exp || framework_failure_ -timeout 10 tail -f fifo > out & pid=$! +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + +timeout 10 tail $fastpoll -f fifo > out & pid=$! check_tail_output() { @@ -40,4 +43,6 @@ compare exp out || fail=1 # Kill the still-running tail, or fail if it's gone. kill $pid || fail=1 +wait $pid + Exit $fail diff --git a/tests/tail-2/retry.sh b/tests/tail-2/retry.sh index 97bd6d3..6aee996 100755 --- a/tests/tail-2/retry.sh +++ b/tests/tail-2/retry.sh @@ -36,27 +36,32 @@ wait4lines_ () [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; } } +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + # === Test: # Retry without --follow results in a warning. touch file tail --retry file > out 2>&1 || fail=1 -[ "$(countlines_)" = 1 ] || fail=1 -grep -F 'tail: warning: --retry ignored' out || fail=1 +[ "$(countlines_)" = 1 ] || { cat out; fail=1; } +grep -F 'tail: warning: --retry ignored' out || { cat out; fail=1; } # === Test: # The same with a missing file: expect error message and exit 1. tail --retry missing > out 2>&1 && fail=1 -[ "$(countlines_)" = 2 ] || fail=1 -grep -F 'tail: warning: --retry ignored' out || fail=1 +[ "$(countlines_)" = 2 ] || { cat out; fail=1; } +grep -F 'tail: warning: --retry ignored' out || { cat out; fail=1; } # === Test: # Ensure that "tail --retry --follow=name" waits for the file to appear. # Clear 'out' so that we can check its contents without races >out || framework_failure_ -timeout 10 tail -s.1 --follow=name --retry missing >out 2>&1 & pid=$! -retry_delay_ wait4lines_ .1 6 1 || fail=1 # Wait for "cannot open" error. -echo "X" > missing || fail=1 -retry_delay_ wait4lines_ .1 6 3 || fail=1 # Wait for the expected output. +timeout 10 tail $fastpoll --follow=name --retry missing >out 2>&1 & pid=$! +# Wait for "cannot open" error. +retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; } +echo "X" > missing || framework_failure_ +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; } kill $pid wait $pid # Expect 3 lines in the output file. @@ -69,10 +74,12 @@ rm -f missing out || framework_failure_ # === Test: # Ensure that "tail --retry --follow=descriptor" waits for the file to appear. # tail-8.21 failed at this (since the implementation of the inotify support). -timeout 10 tail -s.1 --follow=descriptor --retry missing >out 2>&1 & pid=$! -retry_delay_ wait4lines_ .1 6 2 || fail=1 # Wait for "cannot open" error. -echo "X" > missing || fail=1 -retry_delay_ wait4lines_ .1 6 4 || fail=1 # Wait for the expected output. +timeout 10 tail $fastpoll --follow=descriptor --retry missing >out 2>&1 & pid=$! +# Wait for "cannot open" error. +retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; } +echo "X" > missing || framework_failure_ +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; } kill $pid wait $pid # Expect 4 lines in the output file. @@ -87,10 +94,12 @@ rm -f missing out || framework_failure_ # === Test: # Ensure that tail --follow=descriptor --retry exits when the file appears # untailable. Expect exit status 1. -timeout 10 tail -s.1 --follow=descriptor --retry missing >out 2>&1 & pid=$! -retry_delay_ wait4lines_ .1 6 2 || fail=1 # Wait for "cannot open" error. -mkdir missing || fail=1 # Create untailable 'missing'. -retry_delay_ wait4lines_ .1 6 4 || fail=1 # Wait for the expected output. +timeout 10 tail $fastpoll --follow=descriptor --retry missing >out 2>&1 & pid=$! +# Wait for "cannot open" error. +retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; } +mkdir missing || framework_failure_ # Create untailable +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; } wait $pid rc=$? [ "$(countlines_)" = 4 ] || { fail=1; cat out; } diff --git a/tests/tail-2/symlink.sh b/tests/tail-2/symlink.sh index 274df95..9f1a201 100755 --- a/tests/tail-2/symlink.sh +++ b/tests/tail-2/symlink.sh @@ -36,15 +36,20 @@ wait4lines_ () [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; } } +# speedup non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + # Ensure changing targets of cli specified symlinks are handled. # Prior to v8.22, inotify would fail to recognize changes in the targets. # Clear 'out' so that we can check its contents without races. >out || framework_failure_ ln -nsf target symlink || framework_failure_ -timeout 10 tail -s.1 -F symlink >out 2>&1 & pid=$! -retry_delay_ wait4lines_ .1 6 1 || fail=1 # Wait for "cannot open..." -echo "X" > target || fail=1 -retry_delay_ wait4lines_ .1 6 3 || fail=1 # Wait for the expected output. +timeout 10 tail $fastpoll -F symlink >out 2>&1 & pid=$! +# Wait for "cannot open..." +retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; } +echo "X" > target || framework_failure_ +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; } kill $pid wait $pid # Expect 3 lines in the output file. @@ -60,12 +65,15 @@ rm -f target out || framework_failure_ >out || framework_failure_ echo "X1" > target1 || framework_failure_ ln -nsf target1 symlink || framework_failure_ -timeout 10 tail -s.1 -F symlink >out 2>&1 & pid=$! -retry_delay_ wait4lines_ .1 6 1 || fail=1 # Wait for the expected output. +timeout 10 tail $fastpoll -F symlink >out 2>&1 & pid=$! +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; } ln -nsf target2 symlink || framework_failure_ -retry_delay_ wait4lines_ .1 6 2 || fail=1 # Wait for "become inaccess..." -echo "X2" > target2 || fail=1 -retry_delay_ wait4lines_ .1 6 4 || fail=1 # Wait for the expected output. +# Wait for "become inaccess..." +retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; } +echo "X2" > target2 || framework_failure_ +# Wait for the expected output. +retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; } kill $pid wait $pid # Expect 4 lines in the output file. diff --git a/tests/tail-2/tail-n0f.sh b/tests/tail-2/tail-n0f.sh index 26a7b26..f713a12 100755 --- a/tests/tail-2/tail-n0f.sh +++ b/tests/tail-2/tail-n0f.sh @@ -35,11 +35,10 @@ chmod 0 unreadable || framework_failure_ tail -c0 unreadable || fail=1 tail -n0 unreadable || fail=1 -for inotify in ---disable-inotify ''; do +for mode in '' '---disable-inotify'; do for file in empty nonempty; do for c_or_n in c n; do - tail --sleep=4 -${c_or_n} 0 -f $inotify $file & - pid=$! + tail --sleep=4 -${c_or_n} 0 -f $mode $file & pid=$! tail_sleeping() { local delay="$1"; sleep $delay @@ -53,6 +52,7 @@ for inotify in ---disable-inotify ''; do retry_delay_ tail_sleeping .1 4 || { echo $0: process in unexpected state: $state >&2; fail=1; } kill $pid + wait $pid done done done diff --git a/tests/tail-2/wait.sh b/tests/tail-2/wait.sh index 839fcf6..ebae62b 100755 --- a/tests/tail-2/wait.sh +++ b/tests/tail-2/wait.sh @@ -23,30 +23,32 @@ print_ver_ tail touch here || framework_failure_ { touch unreadable && chmod a-r unreadable; } || framework_failure_ +# speedup non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' -for inotify in ---disable-inotify ''; do - timeout 10 tail -s0.1 -f $inotify not_here +for mode in '' '---disable-inotify'; do + timeout 10 tail $fastpoll -f $mode not_here test $? = 124 && fail=1 if test ! -r unreadable; then # can't test this when root - timeout 10 tail -s0.1 -f $inotify unreadable + timeout 10 tail $fastpoll -f $mode unreadable test $? = 124 && fail=1 fi - timeout 1 tail -s0.1 -f $inotify here 2>tail.err + timeout .1 tail $fastpoll -f $mode here 2>tail.err test $? = 124 || fail=1 # 'tail -F' must wait in any case. - timeout 1 tail -s0.1 -F $inotify here 2>>tail.err + timeout .1 tail $fastpoll -F $mode here 2>>tail.err test $? = 124 || fail=1 if test ! -r unreadable; then # can't test this when root - timeout 1 tail -s0.1 -F $inotify unreadable + timeout .1 tail $fastpoll -F $mode unreadable test $? = 124 || fail=1 fi - timeout 1 tail -s0.1 -F $inotify not_here + timeout .1 tail $fastpoll -F $mode not_here test $? = 124 || fail=1 grep -Ev 'inotify (resources exhausted|cannot be used)' tail.err > x @@ -54,13 +56,14 @@ for inotify in ---disable-inotify ''; do compare /dev/null tail.err || fail=1 >tail.err + # Ensure -F never follows a descriptor after rename + # either with tiny or significant delays between operations tail_F() { local delay="$1" touch k || framework_failure_ - tail -s.1 --max-unchanged-stats=2 -F $inotify k > tail.out & - pid=$! + tail $fastpoll -F $mode k > tail.out & pid=$! sleep $delay mv k l sleep $delay @@ -70,11 +73,13 @@ for inotify in ---disable-inotify ''; do echo NO >> l sleep $delay kill $pid + wait $pid rm -f k l - test ! -s tail.out + test -s tail.out } - retry_delay_ tail_F .1 4 || fail=1 + retry_delay_ tail_F 0 1 && { cat tail.out; fail=1; } + retry_delay_ tail_F .2 1 && { cat tail.out; fail=1; } done Exit $fail -- 2.3.4 From fe02153e6475ae52f90cfccae1d1ba24baf35f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Fri, 1 May 2015 18:03:37 +0100 Subject: [PATCH 2/5] tests: avoid hung processes due to gdb SIGCONT handling * tests/tail-2/inotify-race.sh: Add a `wait` to ensure that we reap all background gdb and tail processes. That resulted in the test hanging intermittently and upon investigation was due to gdb intermittently failing to terminate the child process due to receiving a SIGCONT signal. Therefore we avoid using timeout(1) which sends that signal, and instead rely on tail's inbuilt --pid monitoring on a background sleep process. Given this new implementation, the VERY_EXPENSIVE guard was removed. Related issues with this test hanging were previously discussed at: https://lists.gnu.org/archive/html/bug-coreutils/2009-12/msg00025.html --- tests/tail-2/inotify-race.sh | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/tail-2/inotify-race.sh b/tests/tail-2/inotify-race.sh index 2b1655c..d28f898 100755 --- a/tests/tail-2/inotify-race.sh +++ b/tests/tail-2/inotify-race.sh @@ -23,11 +23,6 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail -# Don't run this test by default because sometimes it's skipped as noted below. -# Also gdb has a bug in Debian's gdb-6.8-3 at least that causes it to not -# cleanup and exit correctly when it receives a SIGTERM, thus hanging the test. -very_expensive_ - touch file || framework_failure_ touch tail.out || framework_failure_ @@ -37,29 +32,46 @@ case $(cat gdb.out) in *) skip_ "can't run gdb";; esac +# Break on a line rather than a symbol, to cater for inline functions break_src="$abs_top_srcdir/src/tail.c" break_line=$(grep -n ^tail_forever_inotify "$break_src") || framework_failure_ break_line=$(echo "$break_line" | cut -d: -f1) || framework_failure_ + +# Note we get tail to monitor a background sleep process +# rather than using timeout(1), as timeout sends SIGCONT +# signals to its monitored process, and gdb (7.9 at least) +# has _intermittent_ issues with this. +# Sending SIGCONT resulted in either delayed child termination, +# or no child termination resulting in a hung test. +# See https://sourceware.org/bugzilla/show_bug.cgi?id=18364 + +env sleep 10 & sleep=$! + # See if gdb works and # tail_forever_inotify is compiled and run -timeout 10s gdb -nx --batch-silent \ +gdb -nx --batch-silent \ --eval-command="break $break_line" \ - --eval-command='run -f file' \ + --eval-command="run --pid=$sleep -f file" \ --eval-command='quit' \ - tail < /dev/null > gdb.out 2>&1 || skip_ 'breakpoint not hit' + tail < /dev/null > gdb.out 2>&1 + +kill $sleep || skip_ 'breakpoint not hit' +wait $sleep # FIXME: The above is seen to _intermittently_ fail with: # warning: .dynamic section for "/lib/libc.so.6" is not at the expected address # warning: difference appears to be caused by prelink, adjusting expectations compare /dev/null gdb.out || skip_ "can't set breakpoints in tail" +env sleep 10 & sleep=$! + # Run "tail -f file", stopping to append a line just before # inotify initialization, and then continue. Before the fix, # that just-appended line would never be output. -timeout 10s gdb -nx --batch-silent \ +gdb -nx --batch-silent \ --eval-command="break $break_line" \ - --eval-command='run -f file >> tail.out' \ + --eval-command="run --pid=$sleep -f file >> tail.out" \ --eval-command='shell echo never-seen-with-tail-7.5 >> file' \ --eval-command='continue' \ --eval-command='quit' \ @@ -68,6 +80,14 @@ pid=$! tail --pid=$pid -f tail.out | (read; kill $pid) +# gdb has a bug in Debian's gdb-6.8-3 at least that causes it to not +# cleanup and exit correctly when it receives a SIGTERM, but +# killing sleep, should cause the tail process and thus gdb to exit. +kill $sleep +wait $sleep + +wait $pid + compare /dev/null tail.out && fail=1 Exit $fail -- 2.3.4 From d26a2eff2e4fa4f2f40f617ee4c000eeccbbe5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Fri, 1 May 2015 05:26:38 +0100 Subject: [PATCH 3/5] tests: cleanup background processes upon interruption Reap background processes so that: - Stray processes aren't left on the system - Files aren't held open causing deletion issues on NFS - Partitions used to run the tests from can be unmounted * tests/tail-2/F-vs-missing.sh: Add the `kill && wait` of the background $pid(s) to cleanup_(). * tests/tail-2/F-vs-rename.sh: Likewise. * tests/tail-2/f-vs-rename.sh: Likewise. * tests/tail-2/append-only.sh: Likewise. * tests/tail-2/assert-2.sh: Likewise. * tests/tail-2/assert.sh: Likewise. * tests/tail-2/flush-initial.sh: Likewise. * tests/tail-2/inotify-hash-abuse.sh: Likewise. * tests/tail-2/inotify-hash-abuse2.sh: Likewise. * tests/tail-2/inotify-race.sh: Likewise. * tests/tail-2/inotify-rotate-resources.sh: Likewise. * tests/tail-2/inotify-rotate.sh: Likewise. * tests/tail-2/pid.sh: Likewise. * tests/tail-2/pipe-f2.sh: Likewise. * tests/tail-2/retry.sh: Likewise. * tests/tail-2/symlink.sh: Likewise. * tests/tail-2/tail-n0f.sh: Likewise. * tests/tail-2/wait.sh: Likewise. * tests/cp/existing-perm-race.sh: Likewise. * tests/cp/file-perm-race.sh: Likewise. * tests/cp/parent-perm-race.sh: Likewise. * tests/cp/sparse-to-pipe.sh: Likewise. * tests/dd/stats.sh: Likewise. * tests/du/move-dir-while-traversing.sh: Likewise. * tests/misc/cat-buf.sh: Likewise. * tests/misc/help-version.sh: Likewise. * tests/misc/printf-surprise.sh: Likewise. * tests/misc/sort-compress-proc.sh: Likewise. * tests/misc/sort-spinlock-abuse.sh: Likewise. * tests/misc/stdbuf.sh: Likewise. * tests/misc/tac-continue.sh: Likewise. * tests/misc/timeout-group.sh: Likewise. * tests/mv/i-3.sh: Likewise. * tests/rm/dangling-symlink.sh: Likewise. * tests/rm/isatty.sh: Likewise. * cfg.mk (sc_prohibit_test_background_without_cleanup_): A new syntax-check to ensure cleanup_() is defined when background tasks are created in a test. --- cfg.mk | 7 +++++++ tests/cp/existing-perm-race.sh | 8 +++++--- tests/cp/file-perm-race.sh | 8 +++++--- tests/cp/parent-perm-race.sh | 7 ++++--- tests/cp/sparse-to-pipe.sh | 7 +++++-- tests/dd/stats.sh | 10 +++++++++- tests/du/move-dir-while-traversing.sh | 5 ++++- tests/misc/cat-buf.sh | 6 ++++-- tests/misc/help-version.sh | 7 +++++-- tests/misc/printf-surprise.sh | 5 ++++- tests/misc/sort-compress-proc.sh | 5 ++++- tests/misc/sort-spinlock-abuse.sh | 5 ++++- tests/misc/stdbuf.sh | 11 +++++++---- tests/misc/tac-continue.sh | 10 ++++++++-- tests/misc/timeout-group.sh | 9 +++++---- tests/mv/i-3.sh | 5 ++++- tests/rm/dangling-symlink.sh | 7 ++++--- tests/rm/isatty.sh | 8 +++++--- tests/tail-2/F-vs-missing.sh | 7 ++++--- tests/tail-2/F-vs-rename.sh | 7 ++++--- tests/tail-2/append-only.sh | 3 +++ tests/tail-2/assert-2.sh | 11 ++++++----- tests/tail-2/assert.sh | 13 +++++++------ tests/tail-2/f-vs-rename.sh | 7 ++++--- tests/tail-2/flush-initial.sh | 9 +++++---- tests/tail-2/inotify-hash-abuse.sh | 9 ++++++--- tests/tail-2/inotify-hash-abuse2.sh | 9 ++++++--- tests/tail-2/inotify-race.sh | 9 +++++++-- tests/tail-2/inotify-rotate-resources.sh | 16 ++++++++++------ tests/tail-2/inotify-rotate.sh | 8 +++++--- tests/tail-2/pid.sh | 6 +++--- tests/tail-2/pipe-f2.sh | 9 ++++++--- tests/tail-2/retry.sh | 9 +++++---- tests/tail-2/symlink.sh | 9 +++++---- tests/tail-2/tail-n0f.sh | 6 ++++-- tests/tail-2/wait.sh | 6 ++++-- 36 files changed, 187 insertions(+), 96 deletions(-) diff --git a/cfg.mk b/cfg.mk index 4f222f6..8526853 100644 --- a/cfg.mk +++ b/cfg.mk @@ -487,6 +487,13 @@ sc_prohibit_test_ulimit_without_require_: | sort | uniq -u | grep . && { echo "$(ME): the above test(s)"\ " should match require_ulimit_v_ with ulimit -v" 1>&2; exit 1; } || : +# Ensure that tests call the cleanup_ function if using background processes +sc_prohibit_test_background_without_cleanup_: + @(git grep -El '( &$$|&[^&]*=\$$!)' tests; \ + git grep -l 'cleanup_()' tests | sed p) \ + | sort | uniq -u | grep . && { echo "$(ME): the above test(s)"\ + " should use cleanup_ for background processes" 1>&2; exit 1; } || : + # Ensure that tests call the print_ver_ function for programs which are # actually used in that test. sc_prohibit_test_calls_print_ver_with_irrelevant_argument: diff --git a/tests/cp/existing-perm-race.sh b/tests/cp/existing-perm-race.sh index 30cbc6a..f48369c 100755 --- a/tests/cp/existing-perm-race.sh +++ b/tests/cp/existing-perm-race.sh @@ -37,10 +37,12 @@ chgrp $g1 fifo && chgrp $g2 fifo-copy && chmod g+r fifo-copy || framework-failure +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Copy a fifo's contents. That way, we can examine the # destination permissions before they're finalized. -cp -p --copy-contents fifo fifo-copy & -cp_pid=$! +cp -p --copy-contents fifo fifo-copy & pid=$! ( # Now 'cp' is reading the fifo. Wait for the destination file to @@ -77,7 +79,7 @@ case $mode in *) fail=1;; esac -wait $cp_pid || fail=1 +wait $pid || fail=1 # Check that the final mode and group are right. ls -l -n fifo-copy >ls.out && diff --git a/tests/cp/file-perm-race.sh b/tests/cp/file-perm-race.sh index 5057dae..8197962 100755 --- a/tests/cp/file-perm-race.sh +++ b/tests/cp/file-perm-race.sh @@ -25,10 +25,12 @@ require_local_dir_ umask 022 mkfifo_or_skip_ fifo +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Copy a fifo's contents. That way, we can examine the # destination permissions before they're finalized. -cp -p --copy-contents fifo fifo-copy & -cp_pid=$! +cp -p --copy-contents fifo fifo-copy & pid=$! ( # Now 'cp' is reading the fifo. Wait for the destination file to @@ -51,6 +53,6 @@ case $(cat ls.out) in *) fail=1;; esac -wait $cp_pid || fail=1 +wait $pid || fail=1 Exit $fail diff --git a/tests/cp/parent-perm-race.sh b/tests/cp/parent-perm-race.sh index 7b254b7..c370f9e 100755 --- a/tests/cp/parent-perm-race.sh +++ b/tests/cp/parent-perm-race.sh @@ -26,6 +26,8 @@ umask 002 mkdir mode ownership d || framework_failure_ chmod g+s d 2>/dev/null # The cp test is valid either way. +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } for attr in mode ownership do @@ -33,8 +35,7 @@ do # Copy a fifo's contents. That way, we can examine d/$attr's # state while cp is running. - cp --preserve=$attr -R --copy-contents --parents $attr d & - cp_pid=$! + cp --preserve=$attr -R --copy-contents --parents $attr d & pid=$! ( # Now 'cp' is reading the fifo. @@ -58,7 +59,7 @@ do fail=1;; esac - wait $cp_pid || fail=1 + wait $pid || fail=1 done Exit $fail diff --git a/tests/cp/sparse-to-pipe.sh b/tests/cp/sparse-to-pipe.sh index 8fc9e09..2c3dcef 100755 --- a/tests/cp/sparse-to-pipe.sh +++ b/tests/cp/sparse-to-pipe.sh @@ -21,14 +21,17 @@ print_ver_ cp require_sparse_support_ +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + mkfifo_or_skip_ pipe -timeout 10 cat pipe > copy & +timeout 10 cat pipe > copy & pid=$! truncate -s1M sparse || framework_failure_ cp sparse pipe || fail=1 # Ensure that the cat has completed before comparing. -wait +wait $pid cmp sparse copy || fail=1 diff --git a/tests/dd/stats.sh b/tests/dd/stats.sh index 2d3c3a8..e906c74 100755 --- a/tests/dd/stats.sh +++ b/tests/dd/stats.sh @@ -28,6 +28,14 @@ trap '' $SIGINFO mkfifo_or_skip_ fifo +# Terminate any background processes +cleanup_() +{ + kill $pid 2>/dev/null + kill $pid2 2>/dev/null + wait +} + for open in '' '1'; do # Run dd with the fullblock iflag to avoid short reads # which can be triggered by reception of signals @@ -39,7 +47,7 @@ for open in '' '1'; do # dd will block on open until fifo is opened for reading. # Timeout in case dd goes away erroneously which we check for below. - timeout 10 sh -c 'wc -c < fifo > nwritten' & + timeout 10 sh -c 'wc -c < fifo > nwritten' & pid2=$! # Send lots of signals immediately to ensure dd not killed due # to race setting handler, or blocking on open of fifo. diff --git a/tests/du/move-dir-while-traversing.sh b/tests/du/move-dir-while-traversing.sh index 7d1b58a..67bb345 100755 --- a/tests/du/move-dir-while-traversing.sh +++ b/tests/du/move-dir-while-traversing.sh @@ -71,10 +71,13 @@ for i in $(seq 50); do mkdir -p $t/3/a/b/c/$i/$long || framework_failure_ done +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Prohibit suspension, which could otherwise cause a timeout-induced FP failure. trap '' TSTP -timeout 6 ./inotify-watch-for-dir-access.py $t/3/a/b > start-msg & +timeout 6 ./inotify-watch-for-dir-access.py $t/3/a/b > start-msg & pid=$! # Wait for the watcher to start... nonempty() { test -s start-msg || { sleep $1; return 1; }; } diff --git a/tests/misc/cat-buf.sh b/tests/misc/cat-buf.sh index 68be86e..3138248 100755 --- a/tests/misc/cat-buf.sh +++ b/tests/misc/cat-buf.sh @@ -26,6 +26,8 @@ print_ver_ cat # write separately. mkfifo_or_skip_ fifo +# Terminate any background cp process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } echo 1 > exp @@ -33,9 +35,9 @@ cat_buf_1() { local delay="$1" - dd count=1 if=fifo > out & + dd count=1 if=fifo > out & pid=$! (echo 1; sleep $delay; echo 2) | cat -v > fifo - wait # for dd to complete + wait $pid compare exp out } diff --git a/tests/misc/help-version.sh b/tests/misc/help-version.sh index 64d6e16..e0dd721 100755 --- a/tests/misc/help-version.sh +++ b/tests/misc/help-version.sh @@ -25,6 +25,9 @@ export SHELL . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + expected_failure_status_chroot=125 expected_failure_status_env=125 expected_failure_status_nice=125 @@ -216,8 +219,8 @@ id_setup () { args=-u; } # Use env to avoid invoking built-in sleep of Solaris 11's /bin/sh. kill_setup () { - env sleep 31.5 & - args=$! + env sleep 10m & pid=$! + args=$pid } link_setup () { args="$tmp_in link-target"; } diff --git a/tests/misc/printf-surprise.sh b/tests/misc/printf-surprise.sh index 0af6448..20e6b09 100755 --- a/tests/misc/printf-surprise.sh +++ b/tests/misc/printf-surprise.sh @@ -53,7 +53,10 @@ mkfifo_or_skip_ fifo # http://bugs.debian.org/481543#77 export MALLOC_PERTURB_=0 -head -c 10 fifo > out & +# Terminate any background process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + +head -c 10 fifo > out & pid=$! # Choosing the virtual memory limit, 11000 is enough, but 10000 is too # little and provokes a "memory exhausted" diagnostic on FreeBSD 9.0-p3. diff --git a/tests/misc/sort-compress-proc.sh b/tests/misc/sort-compress-proc.sh index a191a3c..4ad42d5 100755 --- a/tests/misc/sort-compress-proc.sh +++ b/tests/misc/sort-compress-proc.sh @@ -20,6 +20,9 @@ print_ver_ sort expensive_ +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + SORT_FAILURE=2 seq -w 2000 > exp || fail=1 @@ -63,7 +66,7 @@ done # Ignore a random child process created before 'sort' was exec'ed. # This bug was also present in coreutils 8.7. # -( (sleep 1; exec false) & +( (sleep 1; exec false) & pid=$! PRE_COMPRESS='test -f ok || sleep 2' POST_COMPRESS='touch ok' exec sort --compress-program=./compress -S 1k in >out diff --git a/tests/misc/sort-spinlock-abuse.sh b/tests/misc/sort-spinlock-abuse.sh index 31c5aa2..e81abf8 100755 --- a/tests/misc/sort-spinlock-abuse.sh +++ b/tests/misc/sort-spinlock-abuse.sh @@ -29,13 +29,16 @@ very_expensive_ grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null || skip_ 'requires pthreads' +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + seq 100000 > in || framework_failure_ mkfifo_or_skip_ fifo # Arrange for sort to require 8.0+ seconds of wall-clock time, # while actually using far less than 1 second of CPU time. (for i in $(seq 80); do read line; echo $i; sleep .1; done - cat > /dev/null) < fifo & + cat > /dev/null) < fifo & pid=$! # However, under heavy load, it can easily take more than # one second of CPU time, so set a permissive limit: diff --git a/tests/misc/stdbuf.sh b/tests/misc/stdbuf.sh index bb1d40c..428d4a2 100755 --- a/tests/misc/stdbuf.sh +++ b/tests/misc/stdbuf.sh @@ -58,15 +58,18 @@ test $? = 126 || fail=1 stdbuf -o1 no_such # no such command test $? = 127 || fail=1 +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Ensure line buffering stdout takes effect stdbuf_linebuffer() { local delay="$1" printf '1\n' > exp - dd count=1 if=fifo > out 2> err & + dd count=1 if=fifo > out 2> err & pid=$! (printf '1\n'; sleep $delay; printf '2\n') | stdbuf -oL uniq > fifo - wait # for dd to complete + wait $pid compare exp out } @@ -78,9 +81,9 @@ stdbuf_unbuffer() # Ensure un buffering stdout takes effect printf '1\n' > exp - dd count=1 if=fifo > out 2> err & + dd count=1 if=fifo > out 2> err & pid=$! (printf '1\n'; sleep $delay; printf '2\n') | stdbuf -o0 uniq > fifo - wait # for dd to complete + wait $pid compare exp out } diff --git a/tests/misc/tac-continue.sh b/tests/misc/tac-continue.sh index 9078cd5..1389072 100755 --- a/tests/misc/tac-continue.sh +++ b/tests/misc/tac-continue.sh @@ -33,7 +33,13 @@ if ! test -d "$FULL_PARTITION_TMPDIR"; then fi fp_tmp="$FULL_PARTITION_TMPDIR/tac-cont-$$" -cleanup_() { rm -f "$fp_tmp"; } +cleanup_() +{ + # Terminate any background process + # and remove tmp dir + rm -f "$fp_tmp" + kill $pid 2>/dev/null && wait $pid +} # Make sure we can create an empty file there (i.e., no shortage of inodes). if ! touch $fp_tmp; then @@ -54,7 +60,7 @@ seq 5 > in # Give tac a fifo command line argument. # This makes it try to create a temporary file in $TMPDIR. mkfifo_or_skip_ fifo -seq 1000 > fifo & +seq 1000 > fifo & pid=$! TMPDIR=$FULL_PARTITION_TMPDIR tac fifo in >out 2>err && fail=1 cat <<\EOF > exp || fail=1 diff --git a/tests/misc/timeout-group.sh b/tests/misc/timeout-group.sh index 7768292..054c5ae 100755 --- a/tests/misc/timeout-group.sh +++ b/tests/misc/timeout-group.sh @@ -57,15 +57,17 @@ check_timeout_cmd_running() { sleep $delay; return 1; } } +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } # Start above script in its own group. # We could use timeout for this, but that assumes an implementation. -setsid ./group.sh & +setsid ./group.sh & pid=$! # Wait 6.3s for timeout.cmd to start retry_delay_ check_timeout_cmd_running .1 6 || fail=1 # Simulate a Ctrl-C to the group to test timely exit # Note dash doesn't support signalling groups (a leading -) -env kill -INT -- -$! +env kill -INT -- -$pid wait test -e int.received || fail=1 @@ -82,8 +84,7 @@ start=$(date +%s) # Note the first timeout must send a signal that # the second is handling for it to be propagated to the command. # SIGINT, SIGTERM, SIGALRM etc. are implicit. -timeout -sALRM 30 timeout -sINT 25 ./timeout.cmd 20& -pid=$! +timeout -sALRM 30 timeout -sINT 25 ./timeout.cmd 20 & pid=$! # Wait 6.3s for timeout.cmd to start retry_delay_ check_timeout_cmd_running .1 6 || fail=1 kill -ALRM $pid # trigger the alarm of the first timeout command diff --git a/tests/mv/i-3.sh b/tests/mv/i-3.sh index 8cee6bd..77d4a44 100755 --- a/tests/mv/i-3.sh +++ b/tests/mv/i-3.sh @@ -39,6 +39,9 @@ tty=$(readlink -f /dev/stdin) test -r "$tty" 2>&1 \ || skip_ '/dev/stdin is not readable' +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + mv f g < $tty > out 2>&1 & pid=$! # Test for the expected prompt; sleep upon non-match. @@ -54,7 +57,7 @@ check_overwrite_prompt() # Wait for up to 12.7 seconds for the expected prompt. retry_delay_ check_overwrite_prompt .1 7 || { fail=1; cat out; } -kill $pid +cleanup_ mv -f h i > out 2>&1 || fail=1 test -f i || fail=1 diff --git a/tests/rm/dangling-symlink.sh b/tests/rm/dangling-symlink.sh index 5fb50ec..d2e08a8 100755 --- a/tests/rm/dangling-symlink.sh +++ b/tests/rm/dangling-symlink.sh @@ -25,9 +25,10 @@ print_ver_ rm ln -s no-file dangle ln -s / symlink +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } -rm ---presume-input-tty dangle symlink & -pid=$! +rm ---presume-input-tty dangle symlink & pid=$! # The buggy rm (fileutils-4.1.9) would hang here, waiting for input. # Wait up to 3.1s for rm to remove the files @@ -40,6 +41,6 @@ check_files_removed() { } retry_delay_ check_files_removed .1 5 || fail=1 -kill $pid > /dev/null 2>&1 +cleanup_ Exit $fail diff --git a/tests/rm/isatty.sh b/tests/rm/isatty.sh index 61148d5..4eea443 100755 --- a/tests/rm/isatty.sh +++ b/tests/rm/isatty.sh @@ -25,10 +25,12 @@ skip_if_root_ ls /dev/stdin >/dev/null 2>&1 \ || skip_ 'there is no /dev/stdin file' +# Terminate any background processes +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + touch f chmod 0 f -rm ---presume-input-tty f > out 2>&1 & -pid=$! +rm ---presume-input-tty f > out 2>&1 & pid=$! # Wait a second, to give a buggy rm (as in fileutils-4.0.40) # enough time to remove the file. @@ -37,7 +39,7 @@ sleep 1 # The file must still exist. test -f f || fail=1 -kill $pid > /dev/null 2>&1 +cleanup_ # Note the trailing 'x' -- so I don't have to have a trailing # blank in this file :-) diff --git a/tests/tail-2/F-vs-missing.sh b/tests/tail-2/F-vs-missing.sh index 6622f7e..be20ee3 100755 --- a/tests/tail-2/F-vs-missing.sh +++ b/tests/tail-2/F-vs-missing.sh @@ -28,6 +28,9 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -49,9 +52,7 @@ for mode in '' '---disable-inotify'; do tail_re='has appeared' retry_delay_ check_tail_output .1 7 || { echo "$0: file: unexpected delay?"; cat out; fail=1; } - kill $pid - - wait $pid + cleanup_ done diff --git a/tests/tail-2/F-vs-rename.sh b/tests/tail-2/F-vs-rename.sh index e087ec2..ee61a54 100755 --- a/tests/tail-2/F-vs-rename.sh +++ b/tests/tail-2/F-vs-rename.sh @@ -28,6 +28,9 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -74,9 +77,7 @@ for mode in '' '---disable-inotify'; do retry_delay_ tail_f_vs_rename_3 .1 7 || { echo "$0: a: unexpected delay?"; cat out; fail=1; } - kill $pid - - wait $pid + cleanup_ done Exit $fail diff --git a/tests/tail-2/append-only.sh b/tests/tail-2/append-only.sh index 9da10ae..9d20454 100755 --- a/tests/tail-2/append-only.sh +++ b/tests/tail-2/append-only.sh @@ -21,6 +21,9 @@ print_ver_ tail require_root_ +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + chattr_a_works=1 touch f chattr +a f 2>/dev/null || chattr_a_works=0 diff --git a/tests/tail-2/assert-2.sh b/tests/tail-2/assert-2.sh index aaf8785..67804fe 100755 --- a/tests/tail-2/assert-2.sh +++ b/tests/tail-2/assert-2.sh @@ -28,10 +28,12 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' - for mode in '' '---disable-inotify'; do rm -f a foo out touch a || framework_failure_ @@ -41,16 +43,15 @@ for mode in '' '---disable-inotify'; do # Wait up to 12.7s for tail to start. echo x > a || framework_failure_ tail_re='^x$' retry_delay_ check_tail_output .1 7 || - { cat out; fail=1; } + { cat out; fail=1; break; } # Wait up to 12.7s for tail to notice new foo file ok='ok ok ok' echo "$ok" > foo || framework_failure_ tail_re="^$ok$" retry_delay_ check_tail_output .1 7 || - { echo "$0: foo: unexpected delay?"; cat out; fail=1; } + { echo "$0: foo: unexpected delay?"; cat out; fail=1; break; } - kill $pid - wait $pid + cleanup_ done Exit $fail diff --git a/tests/tail-2/assert.sh b/tests/tail-2/assert.sh index 906e364..43bd1e1 100755 --- a/tests/tail-2/assert.sh +++ b/tests/tail-2/assert.sh @@ -33,10 +33,12 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' - for mode in '' '---disable-inotify'; do rm -f a foo out touch a foo || framework_failure_ @@ -46,22 +48,21 @@ for mode in '' '---disable-inotify'; do # Wait up to 12.7s for tail to start. echo x > a || framework_failure_ tail_re='^x$' retry_delay_ check_tail_output .1 7 || - { cat out; fail=1; } + { cat out; fail=1; break; } # Wait 12.7s for this diagnostic: # tail: foo: No such file or directory rm foo || framework_failure_ tail_re='No such file' retry_delay_ check_tail_output .1 7 || - { cat out; fail=1; } + { cat out; fail=1; break; } # Wait up to 12.7s for tail to notice new foo file ok='ok ok ok' echo "$ok" > foo || framework_failure_ tail_re="^$ok$" retry_delay_ check_tail_output .1 7 || - { echo "$0: foo: unexpected delay?"; cat out; fail=1; } + { echo "$0: foo: unexpected delay?"; cat out; fail=1; break; } - kill $pid - wait $pid + cleanup_ done Exit $fail diff --git a/tests/tail-2/f-vs-rename.sh b/tests/tail-2/f-vs-rename.sh index bfd0949..083d714 100755 --- a/tests/tail-2/f-vs-rename.sh +++ b/tests/tail-2/f-vs-rename.sh @@ -28,6 +28,9 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -47,9 +50,7 @@ for mode in '' '---disable-inotify'; do # Wait up to 12.7s for "y" to appear in the output: tail_re='^y$' retry_delay_ check_tail_output .1 7 || { cat out; fail=1; } - kill $pid - - wait $pid + cleanup_ done Exit $fail diff --git a/tests/tail-2/flush-initial.sh b/tests/tail-2/flush-initial.sh index aaeb878..a25bbeb 100755 --- a/tests/tail-2/flush-initial.sh +++ b/tests/tail-2/flush-initial.sh @@ -22,10 +22,13 @@ print_ver_ tail # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + echo line > in || framework_failure_ # Output should be buffered since we're writing to file # so we're depending on the flush to write out -tail $fastpoll -f in > out & tail_pid=$! +tail $fastpoll -f in > out & pid=$! # Wait for 3.1s for the file to be flushed. tail_flush() @@ -37,8 +40,6 @@ tail_flush() } retry_delay_ tail_flush .1 5 || fail=1 -kill $tail_pid - -wait $tail_pid +cleanup_ Exit $fail diff --git a/tests/tail-2/inotify-hash-abuse.sh b/tests/tail-2/inotify-hash-abuse.sh index d8c14c1..6333699 100755 --- a/tests/tail-2/inotify-hash-abuse.sh +++ b/tests/tail-2/inotify-hash-abuse.sh @@ -30,6 +30,9 @@ check_tail_output() { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -58,10 +61,10 @@ for mode in '' '---disable-inotify'; do tail_re='has appeared' retry_delay_ check_tail_output .1 6 || { cat out; fail=1; } - # Kill the working tail, or fail if it has already aborted - kill $pid || fail=1 + # Double check that tail hasn't aborted + kill -0 $pid || fail=1 - wait $pid + cleanup_ done diff --git a/tests/tail-2/inotify-hash-abuse2.sh b/tests/tail-2/inotify-hash-abuse2.sh index b054237..d641c24 100755 --- a/tests/tail-2/inotify-hash-abuse2.sh +++ b/tests/tail-2/inotify-hash-abuse2.sh @@ -20,6 +20,9 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -34,10 +37,10 @@ for mode in '' '---disable-inotify'; do touch f done - # Kill the working tail, or fail if it has already aborted - kill $pid || fail=1 + # Ensure tail hasn't aborted + kill -0 $pid || fail=1 - wait $pid + cleanup_ done Exit $fail diff --git a/tests/tail-2/inotify-race.sh b/tests/tail-2/inotify-race.sh index d28f898..6e1a7fa 100755 --- a/tests/tail-2/inotify-race.sh +++ b/tests/tail-2/inotify-race.sh @@ -23,6 +23,12 @@ . "${srcdir=.}/tests/init.sh"; path_prepend_ ./src print_ver_ tail +# Terminate any background gdb/tail process +cleanup_() { + kill $pid 2>/dev/null && wait $pid + kill $sleep 2>/dev/null && wait $sleep +} + touch file || framework_failure_ touch tail.out || framework_failure_ @@ -75,8 +81,7 @@ gdb -nx --batch-silent \ --eval-command='shell echo never-seen-with-tail-7.5 >> file' \ --eval-command='continue' \ --eval-command='quit' \ - tail < /dev/null > /dev/null 2>&1 & -pid=$! + tail < /dev/null > /dev/null 2>&1 & pid=$! tail --pid=$pid -f tail.out | (read; kill $pid) diff --git a/tests/tail-2/inotify-rotate-resources.sh b/tests/tail-2/inotify-rotate-resources.sh index b65ceb7..7922365 100755 --- a/tests/tail-2/inotify-rotate-resources.sh +++ b/tests/tail-2/inotify-rotate-resources.sh @@ -48,13 +48,17 @@ cleanup_fail() fail=1 } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + fastpoll='-s.1 --max-unchanged-stats=1' -# Normally less than a second is required here, but with heavy load -# and a lot of disk activity, even 20 seconds per grep_timeout is insufficient, -# which leads to this timeout (used as a safety net for cleanup) -# killing tail before processing is completed. touch k || framework_failure_ + +# Note the timeout guard isn't strictly necessary here, +# however without it strace will ignore SIGTERM. +# strace does always honor SIGTERM with the -I2 option, +# though that's not available on RHEL6 for example. timeout 180 strace -e inotify_rm_watch -o strace.out \ tail -F $fastpoll k >> out 2>&1 & pid=$! @@ -92,7 +96,7 @@ for i in $(seq 2); do done test "$reverted_to_polling_" = 1 && skip_ 'inotify resources already depleted' -kill $pid -wait $pid + +cleanup_ Exit $fail diff --git a/tests/tail-2/inotify-rotate.sh b/tests/tail-2/inotify-rotate.sh index be1f07e..638a3bb 100755 --- a/tests/tail-2/inotify-rotate.sh +++ b/tests/tail-2/inotify-rotate.sh @@ -32,11 +32,14 @@ check_tail_output() # Wait up to 25.5 seconds for grep REGEXP 'out' to succeed. grep_timeout() { tail_re="$1" retry_delay_ check_tail_output .1 8; } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + cleanup_fail() { cat out warn_ $1 - kill $pid + cleanup_ fail=1 } @@ -68,8 +71,7 @@ for i in $(seq 50); do # wait for "ok" to appear in 'out' grep_timeout 'ok' || { cleanup_fail 'failed to detect echoed ok'; break; } - kill $pid + cleanup_ done -wait Exit $fail diff --git a/tests/tail-2/pid.sh b/tests/tail-2/pid.sh index 7a49867..b410478 100755 --- a/tests/tail-2/pid.sh +++ b/tests/tail-2/pid.sh @@ -22,6 +22,8 @@ getlimits_ touch empty here || framework_failure_ +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } for mode in '' '---disable-inotify'; do # Use tail itself to create a background process to monitor, @@ -32,9 +34,7 @@ for mode in '' '---disable-inotify'; do timeout 1 tail -f -s.1 --pid=$pid $mode here test $? = 124 || fail=1 - # Cleanup background process - kill $pid - wait $pid + cleanup_ # Ensure that tail --pid=PID exits with success status when PID is dead. # Use an unlikely-to-be-live PID diff --git a/tests/tail-2/pipe-f2.sh b/tests/tail-2/pipe-f2.sh index 4d5fe51..71a081c 100755 --- a/tests/tail-2/pipe-f2.sh +++ b/tests/tail-2/pipe-f2.sh @@ -24,6 +24,9 @@ mkfifo_or_skip_ fifo echo 1 > fifo & echo 1 > exp || framework_failure_ +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -40,9 +43,9 @@ retry_delay_ check_tail_output .1 7 || fail=1 compare exp out || fail=1 -# Kill the still-running tail, or fail if it's gone. -kill $pid || fail=1 +# Ensure tail is still running +kill -0 $pid || fail=1 -wait $pid +cleanup_ Exit $fail diff --git a/tests/tail-2/retry.sh b/tests/tail-2/retry.sh index 6aee996..a1497d9 100755 --- a/tests/tail-2/retry.sh +++ b/tests/tail-2/retry.sh @@ -36,6 +36,9 @@ wait4lines_ () [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # Speedup the non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -62,8 +65,7 @@ retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; } echo "X" > missing || framework_failure_ # Wait for the expected output. retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; } -kill $pid -wait $pid +cleanup_ # Expect 3 lines in the output file. [ "$(countlines_)" = 3 ] || { fail=1; cat out; } grep -F 'cannot open' out || { fail=1; cat out; } @@ -80,8 +82,7 @@ retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; } echo "X" > missing || framework_failure_ # Wait for the expected output. retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; } -kill $pid -wait $pid +cleanup_ # Expect 4 lines in the output file. [ "$(countlines_)" = 4 ] || { fail=1; cat out; } grep -F 'retry only effective for the initial open' out \ diff --git a/tests/tail-2/symlink.sh b/tests/tail-2/symlink.sh index 9f1a201..b5dcbbd 100755 --- a/tests/tail-2/symlink.sh +++ b/tests/tail-2/symlink.sh @@ -36,6 +36,9 @@ wait4lines_ () [ "$(countlines_)" -ge "$elc" ] || { sleep $delay; return 1; } } +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # speedup non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -50,8 +53,7 @@ retry_delay_ wait4lines_ .1 6 1 || { cat out; fail=1; } echo "X" > target || framework_failure_ # Wait for the expected output. retry_delay_ wait4lines_ .1 6 3 || { cat out; fail=1; } -kill $pid -wait $pid +cleanup_ # Expect 3 lines in the output file. [ "$(countlines_)" = 3 ] || { fail=1; cat out; } grep -F 'cannot open' out || { fail=1; cat out; } @@ -74,8 +76,7 @@ retry_delay_ wait4lines_ .1 6 2 || { cat out; fail=1; } echo "X2" > target2 || framework_failure_ # Wait for the expected output. retry_delay_ wait4lines_ .1 6 4 || { cat out; fail=1; } -kill $pid -wait $pid +cleanup_ # Expect 4 lines in the output file. [ "$(countlines_)" = 4 ] || { fail=1; cat out; } grep -F 'become inacce' out || { fail=1; cat out; } diff --git a/tests/tail-2/tail-n0f.sh b/tests/tail-2/tail-n0f.sh index f713a12..ba9f407 100755 --- a/tests/tail-2/tail-n0f.sh +++ b/tests/tail-2/tail-n0f.sh @@ -35,6 +35,9 @@ chmod 0 unreadable || framework_failure_ tail -c0 unreadable || fail=1 tail -n0 unreadable || fail=1 +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + for mode in '' '---disable-inotify'; do for file in empty nonempty; do for c_or_n in c n; do @@ -51,8 +54,7 @@ for mode in '' '---disable-inotify'; do # Wait up to 1.5s for tail to sleep retry_delay_ tail_sleeping .1 4 || { echo $0: process in unexpected state: $state >&2; fail=1; } - kill $pid - wait $pid + cleanup_ done done done diff --git a/tests/tail-2/wait.sh b/tests/tail-2/wait.sh index ebae62b..071d889 100755 --- a/tests/tail-2/wait.sh +++ b/tests/tail-2/wait.sh @@ -23,6 +23,9 @@ print_ver_ tail touch here || framework_failure_ { touch unreadable && chmod a-r unreadable; } || framework_failure_ +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + # speedup non inotify case fastpoll='-s.1 --max-unchanged-stats=1' @@ -72,8 +75,7 @@ for mode in '' '---disable-inotify'; do sleep $delay echo NO >> l sleep $delay - kill $pid - wait $pid + cleanup_ rm -f k l test -s tail.out -- 2.3.4 From 49ab5f73f5d092e7ad83eebba9ce974739544a6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Tue, 5 May 2015 22:11:24 +0100 Subject: [PATCH 4/5] tail: fix inotify startup races The previous fixes to races in the various tail tests, identified actual races in the tail inotify implementation. With --follow=descriptor, if the tailed file was replaced before the inotify watch was added, then any subsequent changes were ignored. Similarly in --follow=name mode, all changes to a new name were effectively ignored if that name was created after the original open() but before the inotify_add_watch(). * src/tail.c (tail_forever_inotify): Fix 3 cases. 1. With -f, don't stop tailing when file removed before watch. 2. With -f, watch right file when file replaced before watch. 3. With -F, inspect correct file when replaced before watch. Existing tests identify these when tail compiled with TAIL_TEST_SLEEP. * tests/tail-2/inotify-rotate-resources.sh: This test also identifies the issue with --follow=name when TAIL_TEST_SLEEP is used. Adjust so the test is immune to such races, and also fail quicker on remote file systems. * tests/tail-2/inotify-race2.sh: A new test using GDB, based on inotify-race.sh, which tests the -F race without needed recompilation with sleeps. * tests/local.mk: Reference the new test. * NEWS: Mention the bug. --- NEWS | 4 ++ src/tail.c | 58 +++++++++++++---- tests/local.mk | 1 + tests/tail-2/inotify-race2.sh | 103 +++++++++++++++++++++++++++++++ tests/tail-2/inotify-rotate-resources.sh | 29 +++++---- 5 files changed, 172 insertions(+), 23 deletions(-) create mode 100755 tests/tail-2/inotify-race2.sh diff --git a/NEWS b/NEWS index fc652f1..da2b53d 100644 --- a/NEWS +++ b/NEWS @@ -45,6 +45,10 @@ GNU coreutils NEWS -*- outline -*- tail -f again follows changes to a file after it's renamed. [bug introduced in coreutils-7.5] + tail --follow no longer misses changes to files if those files were + replaced before inotify watches were created. + [bug introduced in coreutils-7.5] + ** New features chroot accepts the new --skip-chdir option to not change the working directory diff --git a/src/tail.c b/src/tail.c index 19e658b..bc1d04a 100644 --- a/src/tail.c +++ b/src/tail.c @@ -925,12 +925,10 @@ fremote (int fd, const char *name) # define fremote(fd, name) false #endif -/* FIXME: describe */ - +/* open/fstat F->name and handle changes. */ static void recheck (struct File_spec *f, bool blocking) { - /* open/fstat the file and announce if dev/ino have changed */ struct stat new_stats; bool ok = true; bool is_stdin = (STREQ (f->name, "-")); @@ -1312,7 +1310,7 @@ wd_comparator (const void *e1, const void *e2) return spec1->wd == spec2->wd; } -/* Helper function used by 'tail_forever_inotify'. */ +/* Output (new) data for FSPEC->fd. */ static void check_fspec (struct File_spec *fspec, int wd, int *prev_wd) { @@ -1364,12 +1362,18 @@ static bool tail_forever_inotify (int wd, struct File_spec *f, size_t n_files, double sleep_interval) { +# if TAIL_TEST_SLEEP + /* Delay between open() and inotify_add_watch() + to help trigger different cases. */ + xnanosleep (1000000); +# endif unsigned int max_realloc = 3; /* Map an inotify watch descriptor to the name of the file it's watching. */ Hash_table *wd_to_name; bool found_watchable_file = false; + bool tailed_but_unwatchable = false; bool found_unwatchable_dir = false; bool no_inotify_resources = false; bool writer_is_dead = false; @@ -1438,6 +1442,8 @@ tail_forever_inotify (int wd, struct File_spec *f, size_t n_files, if (f[i].wd < 0) { + if (f[i].fd != -1) /* already tailed. */ + tailed_but_unwatchable = true; if (errno == ENOSPC || errno == ENOMEM) { no_inotify_resources = true; @@ -1459,8 +1465,10 @@ tail_forever_inotify (int wd, struct File_spec *f, size_t n_files, /* Linux kernel 2.6.24 at least has a bug where eventually, ENOSPC is always returned by inotify_add_watch. In any case we should revert to polling when there are no inotify resources. Also a specified directory may not - be currently present or accessible, so revert to polling. */ - if (no_inotify_resources || found_unwatchable_dir) + be currently present or accessible, so revert to polling. Also an already + tailed but unwatchable due rename/unlink race, should also revert. */ + if (no_inotify_resources || found_unwatchable_dir + || (follow_mode == Follow_descriptor && tailed_but_unwatchable)) { hash_free (wd_to_name); @@ -1472,12 +1480,38 @@ tail_forever_inotify (int wd, struct File_spec *f, size_t n_files, prev_wd = f[n_files - 1].wd; - /* Check files again. New data can be available since last time we checked - and before they are watched by inotify. */ + /* Check files again. New files or data can be available since last time we + checked and before they are watched by inotify. */ for (i = 0; i < n_files; i++) { - if (!f[i].ignore) - check_fspec (&f[i], f[i].wd, &prev_wd); + if (! f[i].ignore) + { + /* check for new files. */ + if (follow_mode == Follow_name) + recheck (&(f[i]), false); + else if (f[i].fd != -1) + { + /* If the file was replaced in the small window since we tailed, + then assume the watch is on the wrong item (different to + that we've already produced output for), and so revert to + polling the original descriptor. */ + struct stat stats; + + if (stat (f[i].name, &stats) == 0 + && (f[i].dev != stats.st_dev || f[i].ino != stats.st_ino)) + { + error (0, errno, _("%s was replaced"), + quote (pretty_name (&(f[i])))); + hash_free (wd_to_name); + + errno = 0; + return true; + } + } + + /* check for new data. */ + check_fspec (&f[i], f[i].wd, &prev_wd); + } } evlen += sizeof (struct inotify_event) + 1; @@ -1840,7 +1874,7 @@ tail_file (struct File_spec *f, uintmax_t n_units) { struct stat stats; -#if TEST_RACE_BETWEEN_FINAL_READ_AND_INITIAL_FSTAT +#if TAIL_TEST_SLEEP /* Before the tail function provided 'read_pos', there was a race condition described in the URL below. This sleep call made the window big enough to exercise the problem. */ @@ -2286,7 +2320,7 @@ main (int argc, char **argv) if (fflush (stdout) != 0) error (EXIT_FAILURE, errno, _("write error")); - if (!tail_forever_inotify (wd, F, n_files, sleep_interval)) + if (! tail_forever_inotify (wd, F, n_files, sleep_interval)) return EXIT_FAILURE; } error (0, errno, _("inotify cannot be used, reverting to polling")); diff --git a/tests/local.mk b/tests/local.mk index 0d4f9df..0252763 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -161,6 +161,7 @@ check-root: all_tests = \ tests/misc/help-version.sh \ tests/tail-2/inotify-race.sh \ + tests/tail-2/inotify-race2.sh \ tests/misc/invalid-opt.pl \ tests/rm/ext3-perf.sh \ tests/rm/cycle.sh \ diff --git a/tests/tail-2/inotify-race2.sh b/tests/tail-2/inotify-race2.sh new file mode 100755 index 0000000..b5fe7c7 --- /dev/null +++ b/tests/tail-2/inotify-race2.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Ensure that tail does not ignore a tailed-forever file that has been +# replaced between tail's initial read-to-EOF, and when the inotify watches +# are established in tail_forever_inotify. That new file would be ignored +# indefinitely. + +# Copyright (C) 2015 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +print_ver_ tail + +# Terminate any background gdb/tail process +cleanup_() { + kill $pid 2>/dev/null && wait $pid + kill $sleep 2>/dev/null && wait $sleep +} + +touch file || framework_failure_ +touch tail.out || framework_failure_ + +( timeout 10s gdb --version ) > gdb.out 2>&1 +case $(cat gdb.out) in + *'GNU gdb'*) ;; + *) skip_ "can't run gdb";; +esac + +# Break on a line rather than a symbol, to cater for inline functions +break_src="$abs_top_srcdir/src/tail.c" +break_line=$(grep -n ^tail_forever_inotify "$break_src") || framework_failure_ +break_line=$(echo "$break_line" | cut -d: -f1) || framework_failure_ + + +# Note we get tail to monitor a background sleep process +# rather than using timeout(1), as timeout sends SIGCONT +# signals to its monitored process, and gdb (7.9 at least) +# has _intermittent_ issues with this. +# Sending SIGCONT resulted in either delayed child termination, +# or no child termination resulting in a hung test. +# See https://sourceware.org/bugzilla/show_bug.cgi?id=18364 + +env sleep 10 & sleep=$! + +# See if gdb works and +# tail_forever_inotify is compiled and run +gdb -nx --batch-silent \ + --eval-command="break $break_line" \ + --eval-command="run --pid=$sleep -f file" \ + --eval-command='quit' \ + tail < /dev/null > gdb.out 2>&1 + +kill $sleep || skip_ 'breakpoint not hit' +wait $sleep + +# FIXME: The above is seen to _intermittently_ fail with: +# warning: .dynamic section for "/lib/libc.so.6" is not at the expected address +# warning: difference appears to be caused by prelink, adjusting expectations +compare /dev/null gdb.out || skip_ "can't set breakpoints in tail" + +env sleep 10 & sleep=$! + +echo never-seen-with-tail-8.23 > file.new || framework_failure_ + +# Run "tail -f file", stopping to replace with a new file before +# inotify initialization, and then continue. Before the fix, +# changes to the new file would be effectively ignored. +gdb -nx --batch-silent \ + --eval-command="break $break_line" \ + --eval-command="run --pid=$sleep -F file 2>tail.err >>tail.out" \ + --eval-command='shell mv file.new file' \ + --eval-command='continue' \ + --eval-command='quit' \ + tail < /dev/null > /dev/null 2>&1 & pid=$! + +# Note even updating the watched 'file' wouldn't have output +# anything between coreutils 7.5 and 8.23 inclusive as +# The old file descriptor (still held open by tail) was being fstat(). + +tail --pid=$pid -f tail.out | (read; kill $pid) + +# gdb has a bug in Debian's gdb-6.8-3 at least that causes it to not +# cleanup and exit correctly when it receives a SIGTERM, but +# killing sleep, should cause the tail process and thus gdb to exit. +kill $sleep +wait $sleep + +wait $pid + +compare /dev/null tail.out && { cat tail.err; fail=1; } + +Exit $fail diff --git a/tests/tail-2/inotify-rotate-resources.sh b/tests/tail-2/inotify-rotate-resources.sh index 7922365..c13959e 100755 --- a/tests/tail-2/inotify-rotate-resources.sh +++ b/tests/tail-2/inotify-rotate-resources.sh @@ -22,7 +22,11 @@ print_ver_ tail grep '^#define HAVE_INOTIFY 1' "$CONFIG_HEADER" >/dev/null \ || skip_ 'inotify required' -require_strace_ inotify_rm_watch +require_strace_ 'inotify_add_watch,inotify_rm_watch' + +# Quickly skip on remote file systems +df --local . >/dev/null 2>&1 || + skip_ 'inotify not used on remote file system' check_tail_output() { @@ -59,7 +63,7 @@ touch k || framework_failure_ # however without it strace will ignore SIGTERM. # strace does always honor SIGTERM with the -I2 option, # though that's not available on RHEL6 for example. -timeout 180 strace -e inotify_rm_watch -o strace.out \ +timeout 180 strace -e inotify_add_watch,inotify_rm_watch -o strace.out \ tail -F $fastpoll k >> out 2>&1 & pid=$! reverted_to_polling_=0 @@ -67,19 +71,24 @@ for i in $(seq 2); do echo $i echo 'tailed' > k; - # wait for 'tailed' in (after first iteration; new) file and then in 'out' - grep_timeout 'tailed' || { cleanup_fail 'failed to find "tailed"'; break; } + + # Wait for watch on (new) file + strace_re='inotify_add_watch.*MODIFY' retry_delay_ check_strace .1 8 || + no_watch_=1 + + # Assume this is not because we're leaking + # (resources may already be depleted) + # The explicit check for inotify_rm_watch should confirm that. + grep -F 'reverting to polling' out >/dev/null && skip_ 'inotify unused' + + # Otherwise failure is unknown + test "$no_watch_" && { cat out; framework_failure_ 'no inotify_add_watch'; } mv k k.tmp # wait for tail to detect the rename grep_timeout 'inaccessible' || { cleanup_fail 'failed to detect rename'; break; } - # Assume this is not because we're leaking. - # The explicit check for inotify_rm_watch should confirm that. - grep -F 'reverting to polling' out >/dev/null && - { reverted_to_polling_=1; break; } - # Note we strace here rather than consuming all available watches # to be more efficient, but more importantly avoid depleting resources. # Note also available resources can currently be tuned with: @@ -95,8 +104,6 @@ for i in $(seq 2); do >out && >strace.out || framework_failure_ 'failed to reset output files' done -test "$reverted_to_polling_" = 1 && skip_ 'inotify resources already depleted' - cleanup_ Exit $fail -- 2.3.4 From ee89d5a0cc91bee265058af658578bb16258a71d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Mon, 11 May 2015 14:25:19 +0100 Subject: [PATCH 5/5] tail: consistently output all data for truncated files Generally if logs are truncated, they're truncated to 0 length, so output all existing data when our heuristic determines truncation. Note with inotify, truncate() and write() are often determined independently and so all data would be written if that was the case. * src/tail.c (check_fspec): Reset file offset to 0 upon truncation. (tail_forever): Likewise. (recheck): Add a FIXME for the related issue where tail may loses data due to tail discounting older log files too early. * tests/tail-2/truncate.sh: A new test. * tests/local.mk: Reference the new test. * NEWS: Mention the fix. --- NEWS | 3 +++ src/tail.c | 30 ++++++++++++++++++------- tests/local.mk | 1 + tests/tail-2/truncate.sh | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100755 tests/tail-2/truncate.sh diff --git a/NEWS b/NEWS index da2b53d..0fbd4ce 100644 --- a/NEWS +++ b/NEWS @@ -49,6 +49,9 @@ GNU coreutils NEWS -*- outline -*- replaced before inotify watches were created. [bug introduced in coreutils-7.5] + tail --follow consistently outputs all data for a truncated file. + [bug introduced in the beginning] + ** New features chroot accepts the new --skip-chdir option to not change the working directory diff --git a/src/tail.c b/src/tail.c index bc1d04a..dc6304c 100644 --- a/src/tail.c +++ b/src/tail.c @@ -1025,7 +1025,7 @@ recheck (struct File_spec *f, bool blocking) if (f->fd == -1) { error (0, 0, - _("%s has appeared; following end of new file"), + _("%s has appeared; following new file"), quote (pretty_name (f))); } else @@ -1036,7 +1036,7 @@ recheck (struct File_spec *f, bool blocking) /* File has been replaced (e.g., via log rotation) -- tail the new one. */ error (0, 0, - _("%s has been replaced; following end of new file"), + _("%s has been replaced; following new file"), quote (pretty_name (f))); } } @@ -1055,6 +1055,12 @@ recheck (struct File_spec *f, bool blocking) } } + /* FIXME: When ia log is rotated, daemons tend to log to the + old file descriptor until the new file is present and + the daemon is sent a signal. Therefore tail may miss entries + being written to the old file. Perhaps we should keep + the older file open and continue to monitor it until + data is written to a new file. */ if (new_file) { /* Start at the beginning of the file. */ @@ -1193,13 +1199,16 @@ tail_forever (struct File_spec *f, size_t n_files, double sleep_interval) /* reset counter */ f[i].n_unchanged_stats = 0; + /* XXX: This is only a heuristic, as the file may have also + been truncated and written to if st_size >= size + (in which case we ignore new data <= size). */ if (S_ISREG (mode) && stats.st_size < f[i].size) { error (0, 0, _("%s: file truncated"), name); - last = i; - xlseek (fd, stats.st_size, SEEK_SET, name); - f[i].size = stats.st_size; - continue; + /* Assume the file was truncated to 0, + and therefore output all "new" data. */ + xlseek (fd, 0, SEEK_SET, name); + f[i].size = 0; } if (i != last) @@ -1330,12 +1339,17 @@ check_fspec (struct File_spec *fspec, int wd, int *prev_wd) return; } + /* XXX: This is only a heuristic, as the file may have also + been truncated and written to if st_size >= size + (in which case we ignore new data <= size). + Though in the inotify case it's more likely we'll get + separate events for truncate() and write(). */ if (S_ISREG (fspec->mode) && stats.st_size < fspec->size) { error (0, 0, _("%s: file truncated"), name); *prev_wd = wd; - xlseek (fspec->fd, stats.st_size, SEEK_SET, name); - fspec->size = stats.st_size; + xlseek (fspec->fd, 0, SEEK_SET, name); + fspec->size = 0; } else if (S_ISREG (fspec->mode) && stats.st_size == fspec->size && timespec_cmp (fspec->mtime, get_stat_mtime (&stats)) == 0) diff --git a/tests/local.mk b/tests/local.mk index 0252763..8d01064 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -408,6 +408,7 @@ all_tests = \ tests/tail-2/retry.sh \ tests/tail-2/symlink.sh \ tests/tail-2/tail-c.sh \ + tests/tail-2/truncate.sh \ tests/chmod/c-option.sh \ tests/chmod/equal-x.sh \ tests/chmod/equals.sh \ diff --git a/tests/tail-2/truncate.sh b/tests/tail-2/truncate.sh new file mode 100755 index 0000000..e2bc197 --- /dev/null +++ b/tests/tail-2/truncate.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# Ensure all logs are output upon file truncation + +# Copyright (C) 2015 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src +print_ver_ tail + +check_tail_output() +{ + local delay="$1" + grep "$tail_re" out > /dev/null || + { sleep $delay; return 1; } +} + +# Terminate any background tail process +cleanup_() { kill $pid 2>/dev/null && wait $pid; } + +# Speedup the non inotify case +fastpoll='-s.1 --max-unchanged-stats=1' + +for follow in '-f' '-F'; do + for mode in '' '---disable-inotify'; do + rm -f out + seq 10 > f || framework_failure_ + + tail $follow $mode $fastpoll f > out 2>&1 & pid=$! + + # Wait up to 12.7s for tail to start + echo x > $n + tail_re='^10$' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + seq 11 15 > f || framework_failure_ + + # Wait up to 12.7s for new data + tail_re='^15$' retry_delay_ check_tail_output .1 7 || + { cat out; fail=1; } + + cleanup_ + done +done + +Exit $fail -- 2.3.4