escalate signals for unresponsive child processes

.github/workflows/ci_cd.yml

@@ -394,6 +394,30 @@ jobs:
           expected: failure
           actual: ${{ steps.sad_path_timeout.outcome }}
 
+      - name: sad-path (timeout persistent)
+        id: sad_path_timeout_persistent
+        uses: ./
+        continue-on-error: true
+        with:
+          timeout_seconds: 15
+          max_attempts: 2
+          command: |
+            node -e "
+            process.on('SIGTERM', () => console.log('SIGTERM ignored'));
+            setInterval(() => console.log('still running'), 1000);
+            setTimeout(() => {
+              console.log('Process stopping after lack of timeout');
+            }, 300000);
+            "
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: 2
+          actual: ${{ steps.sad_path_timeout_persistent.outputs.total_attempts }}
+      - uses: nick-invision/assert-action@v1
+        with:
+          expected: failure
+          actual: ${{ steps.sad_path_timeout_persistent.outcome }}
+
   ci_integration_timeout_retry_on_timeout:
     name: Run Integration Timeout Tests (retry_on timeout)
     runs-on: ubuntu-latest

src/index.ts

@@ -13,6 +13,8 @@ const OUTPUT_EXIT_ERROR_KEY = 'exit_error';
 
 let exit: number;
 let done: boolean;
+let processExited: boolean;
+let processFinished: boolean;
 
 function getExecutable(inputs: Inputs): string {
   if (!inputs.shell) {
@@ -73,6 +75,8 @@ async function runCmd(attempt: number, inputs: Inputs) {
 
   exit = 0;
   done = false;
+  processExited = false;
+  processFinished = false;
   let timeout = false;
 
   debug(`Running command ${inputs.command} on ${OS} using shell ${executable}`);
@@ -91,9 +95,10 @@ async function runCmd(attempt: number, inputs: Inputs) {
   child.on('exit', (code, signal) => {
     debug(`Code: ${code}`);
     debug(`Signal: ${signal}`);
+    processExited = true;
 
     // timeouts are killed manually
-    if (signal === 'SIGTERM') {
+    if (signal === 'SIGTERM' || signal === 'SIGINT' || signal === 'SIGKILL') {
       return;
     }
 
@@ -109,14 +114,28 @@ async function runCmd(attempt: number, inputs: Inputs) {
     done = true;
   });
 
+  child.on('close', () => {
+    // Occurs on closing of streams and IPC channels.
+    debug(`Process streams closed.`);
+    processFinished = true;
+  });
+
   do {
     await wait(ms.seconds(inputs.polling_interval_seconds));
   } while (Date.now() < end_time && !done);
 
   if (!done && child.pid) {
     timeout = true;
-    kill(child.pid);
+    kill(child.pid, "SIGTERM");
     await retryWait(ms.seconds(inputs.retry_wait_seconds));
+    // If still not done, send SIGINT followed by SIGKILL
+    if (!processExited) {
+      kill(child.pid, "SIGINT");
+      await wait(3000);
+      if (!processFinished){
+        kill(child.pid, "SIGKILL");
+      }
+    }
     throw new Error(`Timeout of ${getTimeout(inputs)}ms hit`);
   } else if (exit > 0) {
     await retryWait(ms.seconds(inputs.retry_wait_seconds));