grub-devel
[Top][All Lists]
Advanced

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[LOCAL-CI v2 1/3] scripts: Add general scripts to aid automated testing


From: Glenn Washburn
Subject: [LOCAL-CI v2 1/3] scripts: Add general scripts to aid automated testing
Date: Tue, 4 Jan 2022 12:00:44 -0600

Signed-off-by: Glenn Washburn <development@efficientek.com>
---
 scripts/ci/build.sh         |  67 ++++++++++++++++++++
 scripts/ci/functions.sh     |  33 ++++++++++
 scripts/ci/make-images.sh   |  86 +++++++++++++++++++++++++
 scripts/ci/process-tests.sh | 111 +++++++++++++++++++++++++++++++++
 scripts/ci/test.sh          | 121 ++++++++++++++++++++++++++++++++++++
 5 files changed, 418 insertions(+)
 create mode 100755 scripts/ci/build.sh
 create mode 100644 scripts/ci/functions.sh
 create mode 100755 scripts/ci/make-images.sh
 create mode 100755 scripts/ci/process-tests.sh
 create mode 100755 scripts/ci/test.sh

diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
new file mode 100755
index 000000000..723dc292f
--- /dev/null
+++ b/scripts/ci/build.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# JOBS: Number of concurrent jobs while building, defaults to number of
+#   processors plus 1, unless number of processors is 1 in which case its 2.
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# INSTALLDIR: Directory to install binaries
+# ARCH: Architecture to build for
+# PLATFORM: Platform to build for
+# CONFIGURE_OPTS: Extra configure options
+# FONT_SOURCE: Path to unicode font named "unifont.<ext>" where ext is one of:
+#   pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# DJVU_FONT_SOURCE: Path to DejaVu font named "DejaVuSans.<ext>" where ext is
+#   one of: pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+
+[ "x${SHELL_TRACE}" = "xy" ] && set -x
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+JOBS=${JOBS:-`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 2`}
+[ "$JOBS" = 1 ] || JOBS=$(($JOBS + 1))
+
+TARGET="${ARCH}-${PLATFORM}"
+
+mkdir -pv ${BUILDDIR}
+
+RET=0
+cd ${BUILDDIR}
+
+if test -f "$FONT_SOURCE"; then
+  ln -svf "$(realpath -e "$FONT_SOURCE")" ./
+fi
+
+if test -f "$DJVU_FONT_SOURCE"; then
+  ln -svf "$(realpath -e "$DJVU_FONT_SOURCE")" ./
+fi
+
+start_log -c -n "configure-$TARGET" "Configuring $TARGET"
+[ -x "$SRCDIR/configure" ] && $SRCDIR/configure --help
+$SRCDIR/configure --target=$ARCH --with-platform=$PLATFORM \
+  --prefix=${INSTALLDIR} $CONFIGURE_OPTS || RET=$?
+end_log -n "configure-$TARGET"
+
+if [ "$RET" -ne 0 ]; then
+  start_log -c -n "showlogs-$TARGET" "Configuring $TARGET failed, showing logs"
+  cat ${BUILDDIR}/config.log
+  end_log -n "showlogs-$TARGET"
+  exit $RET
+fi
+
+start_log -c -n "build-$TARGET" "Building $TARGET"
+make ${JOBS:+-j$JOBS} || RET=$?
+end_log -n "build-$TARGET"
+[ "$RET" -ne 0 ] && exit $RET
+
+start_log -c -n "install-$TARGET" "Installing $TARGET"
+make ${JOBS:+-j$JOBS} install || RET=$?
+end_log -n "install-$TARGET"
+exit $RET
diff --git a/scripts/ci/functions.sh b/scripts/ci/functions.sh
new file mode 100644
index 000000000..2f4cecaa1
--- /dev/null
+++ b/scripts/ci/functions.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+TXT_RED="\e[31m"
+TXT_YELLOW="\e[33m"
+TXT_CLEAR="\e[0m"
+TXT_LOG_COLOR="\e[97;100m"
+
+function start_log() {
+  local LOG_COLLAPSE LOG_NAME
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      -c) LOG_COLLAPSE=1; shift;;
+      -n) LOG_NAME="$2"; shift;;
+      *) [ "$#" -eq 1 ] && LOG_MSG="$1"; shift;;
+    esac
+  done
+
+  echo -e "Start:${LOG_NAME} ${LOG_MSG}"
+}
+
+function end_log() {
+  local LOG_NAME
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      -n) LOG_NAME=$2; shift;;
+      *) shift;;
+    esac
+  done
+
+  echo -e "End:${LOG_NAME}"
+}
+
+:;
diff --git a/scripts/ci/make-images.sh b/scripts/ci/make-images.sh
new file mode 100755
index 000000000..d156af46f
--- /dev/null
+++ b/scripts/ci/make-images.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to test
+# MAKE_ALL_IMAGE_TARGETS: If set to 'y', then all image targets, even disabled
+#   ones, will be run.
+# DISABLED_IMAGES: String of target formats to disable when building grub
+#   images delimited by new lines.
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# NUM_FAILED_LOG_LINES: Set to integer number of log lines to display from the
+#   end of a failed log file
+
+if [ "x${SHELL_TRACE}" = "xy" ]; then
+  # If we export SHELL_OPTS, then all shells will be traced, most of which we 
do not want
+  SHELL_OPTS="-x"
+  set -x
+fi
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+start_log -c -n "images-$TARGET" "Making images for $TARGET"
+
+if [ ! -x "${BUILDDIR}/grub-mkimage" ]; then
+  echo "Aborting, no grub-mkimage found in builddir"
+  exit 1
+fi
+
+function build_tformat() {
+  TARGET=$1
+  FORMATS=$2
+  for fmt in $FORMATS; do
+    if [ -z "${fmt%~*}" ]; then
+      echo "${TARGET}"
+    else
+      echo "${TARGET}-${fmt}"
+    fi
+  done
+}
+
+tformats="~"
+case "$TARGET" in
+  mipsel-loongson)
+    tformats="mipsel-loongson-elf mipsel-yeeloong-flash 
mipsel-fuloong2f-flash" ;;
+  *)
+    case "$TARGET" in
+      i386-pc) tformats="~ pxe eltorito" ;;
+      sparc64-ieee1275) tformats="aout cdcore raw" ;;
+      mips-qemu_mips | \
+      mipsel-qemu_mips) tformats="elf flash" ;;
+      arm-coreboot) tformats="vexpress veyron" ;;
+    esac
+    tformats=$(build_tformat "${TARGET}" "$tformats")
+    ;;
+esac
+
+RET=0
+mkdir -pv "${BUILDDIR}/images"
+for tfmt in $tformats; do
+  if [ "x${MAKE_ALL_IMAGE_TARGETS}" != "xy" ] && ( echo "${DISABLED_IMAGES}" | 
grep -qE "^\s*${tfmt}$" ); then
+    echo "Image build disabled for target format $tfmt"
+    continue
+  fi
+
+  echo "Making image for target format: ${tfmt}"
+  ${BUILDDIR}/grub-mkimage -v -p / -O "${tfmt}" -o 
"${BUILDDIR}/images/grub-${tfmt}.img" echo reboot normal > 
"${BUILDDIR}/grub-mkimage-${tfmt}.log" 2>&1 || _RET=$?
+  [ "${_RET:-0}" -ne 0 ] && RET=$_RET
+
+  if [ "$RET" -ne 0 ]; then
+    echo -e "${TXT_RED}Failed to build image for target format ${tfmt}: 
${TESTNAME}$TXT_CLEAR"
+    echo -e -n "$TXT_LOG_COLOR"
+    echo "Last ${NUM_FAILED_LOG_LINES} lines of grub-mkimage verbose log"
+    tail -n "${NUM_FAILED_LOG_LINES}" "${BUILDDIR}/grub-mkimage-${tfmt}.log"
+    echo -e -n "$TXT_CLEAR"
+  fi
+done
+
+end_log -n "images-$TARGET"
+exit $RET
diff --git a/scripts/ci/process-tests.sh b/scripts/ci/process-tests.sh
new file mode 100755
index 000000000..9bf763502
--- /dev/null
+++ b/scripts/ci/process-tests.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to process
+# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be
+#   processed.
+# DISABLED_TESTS: String of disabled tests delimited by new lines.
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# EXPECTED_FAILURES: Multi-line string where each line contains a testname or
+#   target:testname indicating an expected failure. Lines whose first non-space
+#   character is '#' are ignored.
+# EXPECTED_FUNCTIONAL_FAILURES: Same format as EXPECTED_FAILURES applied to the
+#   functional test suite (grub_func_test).
+# FAIL_ON_HARD_ERRORS: Set to 'y' to fail on hard as well as normal errors.
+# IGNORE_TIMEDOUT_TEST_FAILURE: Set to 'y' to ignore test failures due to
+#   test timing out.
+# NUM_FAILED_LOG_LINES: Set to integer number of log lines to display from the
+#   end of a failed log file
+
+[ "x${SHELL_TRACE}" = "xy" ] && set -x
+
+[ -f "$(dirname "$0")/functions.sh" ] &&
+. "$(dirname "$0")/functions.sh"
+
+[ -f "$(dirname "$0")/functions.$CI_TYPE.sh" ] &&
+. "$(dirname "$0")/functions.$CI_TYPE.sh"
+
+# TARGET="${ARCH}-${PLATFORM}"
+NUM_FAILED_LOG_LINES=${NUM_FAILED_LOG_LINES:-100}
+
+if [ -f ${BUILDDIR}/test.success ]; then
+  echo "Not processing test logs because make check ran successfully"
+  exit 0
+elif [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep 
-qE "^\s*${TARGET}$" ); then
+  echo "No test output processing because testing was disabled"
+  exit 0
+fi;
+
+start_log -c -n "process-test-$TARGET" "Processing test output of $TARGET"
+: >${BUILDDIR}/test-results.log
+if [ -f ${BUILDDIR}/test-suite.log ]; then
+  ( grep -E "^(FAIL|ERROR|SKIP):" ${BUILDDIR}/test-suite.log || ':' ) |
+  while read STATUS TESTNAME; do
+    STATUS=${STATUS%%:}
+
+    if [ "$STATUS" = "SKIP" ]; then
+      TYPE=skip
+    elif tail -n1 "${BUILDDIR}/${TESTNAME}.log" | grep -qE 'exit status: 
(124|137)'; then
+      if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" = "xy" ]; then
+        echo -e "${TXT_RED}Ignoring Timed-out test: ${TESTNAME}$TXT_CLEAR"
+        echo -e -n "$TXT_LOG_COLOR"
+        echo "Test failed due to a timeout. This is likely due to"
+        echo "insufficient host resources, and NOT a real failure."
+        echo -e -n "$TXT_CLEAR"
+      fi
+      TYPE=timeout
+    elif [ "$STATUS" = "ERROR" ]; then
+      TYPE=error
+    elif echo "${EXPECTED_FAILURES}" | grep -qE 
"^\s*(${TESTNAME}|${TARGET}:${TESTNAME})$"; then
+      echo "Expected failed test: $TESTNAME"
+      TYPE=expected
+    # If any unexpected failures in the functional tests, count a failure in
+    # grub_func_test as a true failure.
+    elif [ "$TESTNAME" = "grub_func_test" ]; then
+      grep -E ': FAIL' ${BUILDDIR}/${TESTNAME}.log | sort -u |
+      while read FTESTNAME STATUS; do
+        FTESTNAME=${FTESTNAME%:*}
+        if echo "${EXPECTED_FUNCTIONAL_FAILURES}" | grep -qE 
"^\s*(${FTESTNAME}|${TARGET}:${FTESTNAME})$"; then
+          echo "Expected failed functional test: $FTESTNAME"
+          TYPE=expected
+        else
+          echo -e "${TXT_RED}Unexpected failed functional test: 
${FTESTNAME}$TXT_CLEAR"
+          TYPE=unexpected
+        fi
+        echo "${TARGET}:${TYPE}:${TESTNAME}:${FTESTNAME}" >> 
${BUILDDIR}/test-results.log
+      done
+      continue
+    else
+      echo -e "${TXT_RED}Unexpected failed test: ${TESTNAME}$TXT_CLEAR"
+      echo -e -n "$TXT_LOG_COLOR"
+      echo "Last ${NUM_FAILED_LOG_LINES} lines of ${BUILDDIR}/${TESTNAME}.log"
+      tail -n "${NUM_FAILED_LOG_LINES}" "${BUILDDIR}/${TESTNAME}.log"
+      echo -e -n "$TXT_CLEAR"
+      TYPE=unexpected
+    fi
+    echo "${TARGET}:${TYPE}:${TESTNAME}" >> ${BUILDDIR}/test-results.log
+  done
+else
+  echo "No success canary and no test-suite.log, perhaps tests were not run or 
was killed."
+  RET=2
+fi
+
+# If there are any unexpected errors return exit with error
+FAILCOND="unexpected"
+if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then
+  FAILCOND="${FAILCOND}|error"
+fi;
+if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" != "xy" ]; then
+  FAILCOND="${FAILCOND}|timeout"
+fi;
+FAILCOND="${TARGET}:(${FAILCOND})"
+
+if grep -E -qs "${FAILCOND}" ${BUILDDIR}/test-results.log; then
+  RET=1
+fi
+end_log -n "process-test-$TARGET"
+exit ${RET:-0}
diff --git a/scripts/ci/test.sh b/scripts/ci/test.sh
new file mode 100755
index 000000000..833912587
--- /dev/null
+++ b/scripts/ci/test.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Environment variables
+# SRCDIR: Path to source files
+# BUILDDIR: Directory in which to place the build
+# TARGET: Target to test
+# TEST_ALL_TARGETS: If set to 'y', then all tests, even disabled ones will be
+#   run.
+# DISABLED_TESTS: String of disabled tests delimited by new lines.
+# TESTS_TIMEOUT: Maximum timeout to allow for 'make check'. Set this slightly
+#   below total job timeout to allow for cleanup and log packaging code.
+# TEST_VERBOSITY: Number to set verbosity level to
+# TEST_DATA_PREFIX: Filename prefix to use when saving test data. If not set,
+#   test data will not be saved.
+# STRACE_TESTS: Set to 'y' to strace each test
+# SHELL_TRACE: Set to 'y' to enable shell tracing
+# JOBS: Number of concurrent jobs while building, defaults to number of
+#   processors plus 1, unless number of processors is 1 in which case its 2.
+
+
+if [ "x${SHELL_TRACE}" = "xy" ]; then
+  # If we export SHELL_OPTS, then all shells will be traced, most of which we 
do not want
+  SHELL_OPTS="-x"
+  set -x
+fi
+
+[ -f "${0%/*}/functions.sh" ] &&
+. "${0%/*}/functions.sh"
+
+[ -f "${0%/*}/functions.$CI_TYPE.sh" ] &&
+. "${0%/*}/functions.$CI_TYPE.sh"
+
+if [ "x${TEST_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_TESTS}" | grep -qE 
"^\s*${TARGET}$" ); then
+  echo "Tests are disabled for target $TARGET"
+  exit 0
+fi
+
+start_log -c -n "test-$TARGET" "Testing $TARGET"
+
+if [ "${TEST_VERBOSITY:-0}" -gt 0 ]; then
+  export GRUB_SHELL_DEFAULT_DEBUG=${TEST_VERBOSITY}
+  export GRUB_TEST_DEFAULT_DEBUG=${TEST_VERBOSITY}
+fi
+
+JOBS="${JOBS:-`getconf _NPROCESSORS_ONLN 2> /dev/null || echo 2`}"
+TESTTMPDIR="${TESTTMPDIR:-${TMPDIR:-/tmp}/grub-test-$TARGET}"
+COMP=xz
+STRACE_LOG="${BUILDDIR}/xxx.strace.$COMP"
+
+if [ "x${STRACE_TESTS}" = "xy" ]; then
+  STRACE="strace -y -yy -f -v -s4096 -o >($COMP -9 > $STRACE_LOG)"
+fi
+
+cat >${BUILDDIR}/log-tester.sh <<EOF
+#!$SHELL $SHELL_OPTS
+set -e
+
+STRACE_LOG="$STRACE_LOG"
+STRACE="$STRACE"
+TESTNAME="\${1##*/}"
+export SHELL_OPTS="$SHELL_OPTS"
+export TMPDIR=$TESTTMPDIR/\$TESTNAME
+mkdir -p "\$TMPDIR"
+
+# if not a shell script, run normally
+if [ "\$(head -c2 \$1)" = "#!" ]; then
+  BUILD_SHEBANG="\$(head -n1 "${BUILDDIR}/grub-shell" | tail -c+3 | sed -E 
's|^\\s*||')"
+  TEST_SHELL=\$(head -n1 "\$1" | tail -c+3 | sed -E 's|^\\s*||')
+  # Only turn on tracing if the shell is the one used by grub-shell
+  if test "\${TEST_SHELL}" = "\${BUILD_SHEBANG}"; then
+    if [ '(' "\${TEST_VERBOSITY:-0}" -gt 0 -o "x\${SHELL_TRACE}" = "xy" ')' \\
+        -a '(' -z "\$SHELL_OPTS" -o -n "\${SHELL_OPTS##*-x*}" ')' ]; then
+      SHELL_OPTS="\$SHELL_OPTS -x"
+    fi
+  fi
+fi
+
+TSTART=\$(date +%s)
+eval \$(echo -n "\$STRACE" | sed "s/xxx/\$TESTNAME/g") "\$TEST_SHELL" "\$@" || 
RET=\$?
+TEND=\$(date +%s)
+echo "Test duration in seconds: \$((TEND - TSTART))"
+
+if [ "\${RET:-0}" -eq "0" ]; then
+  rm -rf \$(echo -n "\$STRACE_LOG" | sed "s/xxx/\$TESTNAME/g") \$TMPDIR
+elif [ "\${RET:-0}" -eq "124" ] || [ "\${RET:-0}" -eq "137" ]; then
+  # This is these exitcodes are for a timed out process when using timeout
+  # In this case return a hard error, otherwise it will be converted to a
+  # normal error.
+  exit 99
+fi
+
+exit \$RET
+EOF
+export LOG_COMPILER="${BUILDDIR}/log-tester.sh"
+chmod +x ${BUILDDIR}/log-tester.sh
+echo -n -e "${TXT_YELLOW}"
+cat ${BUILDDIR}/log-tester.sh
+echo -n -e "${TXT_CLEAR}"
+
+# Setting debug to greater than 9 turns on the graphical qemu window.
+# But we want to run in headless environments, so make sure debug values
+# do not go above 9.
+[ "${GRUB_SHELL_DEFAULT_DEBUG:-0}" -gt 9 ] && GRUB_SHELL_DEFAULT_DEBUG=9
+[ "${GRUB_TEST_DEFAULT_DEBUG:-0}" -gt 9 ] && GRUB_TEST_DEFAULT_DEBUG=9
+
+TIMEOUT=${TESTS_TIMEOUT}
+
+RET=0
+${TIMEOUT:+timeout $TIMEOUT} make -C ${BUILDDIR} ${JOBS:+-j$JOBS} SUBDIRS= 
check || RET=$?
+[ "$RET" -eq 124 -o "$RET" -eq 137 ] && echo "ERROR: make check timed out"
+
+end_log -n "test-$TARGET"
+
+[ "$RET" -eq 0 ] && touch ${BUILDDIR}/test.success || :
+if [ -n "$TEST_DATA_PREFIX" ]; then
+  tar -S -cJf "${TEST_DATA_PREFIX}.tar.xz" -C "${TESTTMPDIR%/*}" 
"${TESTTMPDIR##*/}"
+fi
+
+exit $RET
-- 
2.27.0




reply via email to

[Prev in Thread] Current Thread [Next in Thread]