ci: Add timeout command to test execution.

The timeout command allows us to specify a timeout for each job in CI. This is handy because there might be a test that is not able to finish fro various reasons, last time it was forking nc process. At the same time having timeout over the make command allows us to preserve artifacts even if the timeout is reached so it might be easier to debug what is happening. Signed-off-by: Ales Musil <amusil@redhat.com>
almusil · Oct 10, 2024 · 9fbab00 · 9fbab00
1 parent 6d2243a
commit 9fbab00
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 8 deletions.
diff --git a/.ci/ci.sh b/.ci/ci.sh
@@ -20,6 +20,7 @@ CONTAINER_CMD=${CONTAINER_CMD:-podman}
 CONTAINER_WORKSPACE="/workspace"
 CONTAINER_WORKDIR="/workspace/ovn-tmp"
 IMAGE_NAME=${IMAGE_NAME:-"ovn-org/ovn-tests"}
+TIMEOUT=${TIMEOUT:-"0"}
 
 # Test variables
 ARCH=${ARCH:-$(uname -m)}
@@ -100,7 +101,8 @@ function run_tests() {
         && \
         ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
         TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
-        RECHECK=$RECHECK UNSTABLE=$UNSTABLE ./.ci/linux-build.sh
+        RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \
+        ./.ci/linux-build.sh
     "
 }
 
@@ -115,7 +117,8 @@ function check_clang_version_ge() {
 }
 
 options=$(getopt --options "" \
-    --long help,shell,archive-logs,jobs:,ovn-path:,ovs-path:,image-name:\
+    --long help,shell,archive-logs,jobs:,ovn-path:,ovs-path:,\
+    image-name:,timeout:\
     -- "${@}")
 eval set -- "$options"
 while true; do
@@ -142,11 +145,15 @@ while true; do
     --archive-logs)
         archive_logs="1"
         ;;
+    --timeout)
+        shift
+        TIMEOUT="$1"
+        ;;
     --help)
         set +x
         printf "$0 [--shell] [--help] [--archive-logs] [--jobs=<JOBS>] "
         printf "[--ovn-path=<OVN_PATH>] [--ovs-path=<OVS_PATH>] "
-        printf "[--image-name=<IMAGE_NAME>]\n"
+        printf "[--image-name=<IMAGE_NAME>] [--timeout=<TIMEOUT>]\n"
         exit
         ;;
     --)

diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh
@@ -10,6 +10,7 @@ OVN_CFLAGS=""
 OPTS="$OPTS --enable-Werror"
 JOBS=${JOBS:-"-j4"}
 RECHECK=${RECHECK:-"no"}
+TIMEOUT=${TIMEOUT:="0"}
 
 function install_dpdk()
 {
@@ -105,7 +106,8 @@ function configure_clang()
 
 function run_tests()
 {
-    if ! make distcheck CFLAGS="${COMMON_CFLAGS} ${OVN_CFLAGS}" $JOBS \
+    if ! timeout $TIMEOUT make distcheck \
+        CFLAGS="${COMMON_CFLAGS} ${OVN_CFLAGS}" $JOBS \
         TESTSUITEFLAGS="$JOBS $TEST_RANGE" RECHECK=$RECHECK \
         SKIP_UNSTABLE=$SKIP_UNSTABLE
     then
@@ -147,8 +149,9 @@ function run_system_tests()
     local type=$1
     local log_file=$2
 
-    if ! sudo make $JOBS $type TESTSUITEFLAGS="$TEST_RANGE" \
-            RECHECK=$RECHECK SKIP_UNSTABLE=$SKIP_UNSTABLE; then
+    if ! sudo timeout $TIMEOUT make $JOBS $type \
+            TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \
+            SKIP_UNSTABLE=$SKIP_UNSTABLE; then
         # $log_file is necessary for debugging.
         cat tests/$log_file
         return 1

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -168,11 +168,11 @@ jobs:
 
     - name: build
       if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
-      run: sudo -E ./.ci/ci.sh --archive-logs
+      run: sudo -E ./.ci/ci.sh --archive-logs --timeout=1h
 
     - name: build
       if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') }}
-      run: ./.ci/ci.sh --archive-logs
+      run: ./.ci/ci.sh --archive-logs --timeout=1h
 
     - name: upload logs on failure
       if: failure() || cancelled()