diff --git a/Jenkinsfile b/Jenkinsfile
index 017b5ac74e21b6..963a8f134783d7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -172,68 +172,95 @@ node {
if (env.BRANCH_NAME == 'jenkins_stress_NOT_NOW') {
sh '''#!/bin/bash
-
- # get crumb for CSRF
- COOKIE_JAR=/tmp/cookies
- CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
-
- N=5
- FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber)
- LAST_RUN=$((FIRST_RUN+N))
-
- for i in $(seq $FIRST_RUN $LAST_RUN);
- do
- # start build i
- curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec
- done
-
- while true; do
- sleep 5
-
- count=0
- FAIL=()
- PASS=()
-
- for i in $(seq $FIRST_RUN $LAST_RUN);
- do
- RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json)
- HTTP_CODE=$(tail -n1 <<< "$RES")
- JSON=$(sed '$ d' <<< "$RES")
-
- if [[ $HTTP_CODE == "200" ]]; then
- STILL_RUNNING=$(echo $JSON | jq .inProgress)
- if [[ $STILL_RUNNING == "true" ]]; then
- echo "build $i still running"
- continue
- fi
-
- RESULT=$(echo $JSON | jq .result)
- ((count++))
- echo "build $i $RESULT"
-
- if [[ $RESULT == '"SUCCESS"' ]]; then
- PASS+=($i)
- elif [[ $RESULT == '"FAILURE"' ]]; then
- FAIL+=($i)
- elif [[ $RESULT == '"ABORTED"' ]]; then
- FAIL+=($i)
- else
- FAIL+=($i)
- fi
-
- else
- echo "Error getting status of build $i"
- fi
-
- done
-
- if [[ $count -eq $N ]]; then
- echo "FAIL : ${FAIL[@]}"
- echo "PASS : ${PASS[@]}"
- break
+ # get crumb for CSRF
+ COOKIE_JAR=/tmp/cookies
+ CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+ N=5
+ FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/api/json | jq .nextBuildNumber)
+ LAST_RUN=$((FIRST_RUN+N))
+
+ for i in $(seq $FIRST_RUN $LAST_RUN);
+ do
+ # start build i
+ curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/build?delay=0sec
+ done
+
+ while true; do
+ sleep 60
+
+ count=0
+ for i in $(seq $FIRST_RUN $LAST_RUN);
+ do
+ RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/api/json)
+ HTTP_CODE=$(tail -n1 <<< "$RES")
+ JSON=$(sed '$ d' <<< "$RES")
+
+ if [[ $HTTP_CODE == "200" ]]; then
+ STILL_RUNNING=$(echo $JSON | jq .inProgress)
+ if [[ $STILL_RUNNING == "true" ]]; then
+ echo "build $i still running"
+ continue
fi
-
- done
+ ((count++))
+ else
+ echo "Error getting status of build $i"
+ fi
+ done
+
+ if [[ $count -eq $N ]]; then
+ break
+ fi
+ done
+
+
+ STAGES_NAMES=()
+ while read stage; do
+ STAGES_NAMES[$index]=$stage
+ ((index++))
+ done < <(curl -s -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/lastBuild/wfapi/ | jq .stages[].name)
+ STAGES_COUNT=${#STAGES_NAMES[@]}
+
+ STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
+ STAGES_FAILURES_LOGS=()
+
+ for i in $(seq $FIRST_RUN $LAST_RUN);
+ do
+
+ index=0
+ while read result; do
+ if [[ $result != '"SUCCESS"' ]]; then
+ STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1))
+ STAGES_FAILURES_LOGS[$index]="${STAGES_FAILURES_LOGS[$index]}Log for run #$(($i-$FIRST_RUN))
"
+ fi
+ ((index++))
+ done < <(curl https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/wfapi/ | jq .stages[].status)
+
+ done
+
+ TABLE="
Stage | ✅ Passing | ❌ Failure Details | |
---|---|---|---|
${STAGES_NAMES[$i]} | " + TABLE="${TABLE}$((100-(${STAGES_FAILURES[$i]}*100/$N)))% | " + if [[ ${STAGES_FAILURES[$i]} -eq 0 ]]; then + TABLE="${TABLE}" + else + TABLE="${TABLE} | ${STAGES_FAILURES_LOGS[$i]} | "
+ fi
+ TABLE="${TABLE}