diff --git a/Jenkinsfile b/Jenkinsfile index 017b5ac74e21b6..963a8f134783d7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -172,68 +172,95 @@ node { if (env.BRANCH_NAME == 'jenkins_stress_NOT_NOW') { sh '''#!/bin/bash - - # get crumb for CSRF - COOKIE_JAR=/tmp/cookies - CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') - - N=5 - FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/api/json | jq .nextBuildNumber) - LAST_RUN=$((FIRST_RUN+N)) - - for i in $(seq $FIRST_RUN $LAST_RUN); - do - # start build i - curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/build?delay=0sec - done - - while true; do - sleep 5 - - count=0 - FAIL=() - PASS=() - - for i in $(seq $FIRST_RUN $LAST_RUN); - do - RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/__jenkins_stress/$i/api/json) - HTTP_CODE=$(tail -n1 <<< "$RES") - JSON=$(sed '$ d' <<< "$RES") - - if [[ $HTTP_CODE == "200" ]]; then - STILL_RUNNING=$(echo $JSON | jq .inProgress) - if [[ $STILL_RUNNING == "true" ]]; then - echo "build $i still running" - continue - fi - - RESULT=$(echo $JSON | jq .result) - ((count++)) - echo "build $i $RESULT" - - if [[ $RESULT == '"SUCCESS"' ]]; then - PASS+=($i) - elif [[ $RESULT == '"FAILURE"' ]]; then - FAIL+=($i) - elif [[ $RESULT == '"ABORTED"' ]]; then - FAIL+=($i) - else - FAIL+=($i) - fi - - else - echo "Error getting status of build $i" - fi - - done - - if [[ $count -eq $N ]]; then - echo "FAIL : ${FAIL[@]}" - echo "PASS : ${PASS[@]}" - break + # get crumb for CSRF + COOKIE_JAR=/tmp/cookies + CRUMB=$(curl --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') + + N=5 + FIRST_RUN=$(curl --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/api/json | jq .nextBuildNumber) + LAST_RUN=$((FIRST_RUN+N)) + + for i in $(seq $FIRST_RUN $LAST_RUN); + do + # start build i + curl --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/build?delay=0sec + done + + while true; do + sleep 60 + + count=0 + for i in $(seq $FIRST_RUN $LAST_RUN); + do + RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/api/json) + HTTP_CODE=$(tail -n1 <<< "$RES") + JSON=$(sed '$ d' <<< "$RES") + + if [[ $HTTP_CODE == "200" ]]; then + STILL_RUNNING=$(echo $JSON | jq .inProgress) + if [[ $STILL_RUNNING == "true" ]]; then + echo "build $i still running" + continue fi - - done + ((count++)) + else + echo "Error getting status of build $i" + fi + done + + if [[ $count -eq $N ]]; then + break + fi + done + + + STAGES_NAMES=() + while read stage; do + STAGES_NAMES[$index]=$stage + ((index++)) + done < <(curl -s -H "$CRUMB" https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/lastBuild/wfapi/ | jq .stages[].name) + STAGES_COUNT=${#STAGES_NAMES[@]} + + STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done)) + STAGES_FAILURES_LOGS=() + + for i in $(seq $FIRST_RUN $LAST_RUN); + do + + index=0 + while read result; do + if [[ $result != '"SUCCESS"' ]]; then + STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1)) + STAGES_FAILURES_LOGS[$index]="${STAGES_FAILURES_LOGS[$index]}Log for run #$(($i-$FIRST_RUN))
" + fi + ((index++)) + done < <(curl https://jenkins.comma.life/job/openpilot/job/jenkins_test_runner/$i/wfapi/ | jq .stages[].status) + + done + + TABLE="" + for i in $(seq 0 $(($STAGES_COUNT-1))); + do + TABLE="${TABLE}" + TABLE="${TABLE}" + TABLE="${TABLE}" + if [[ ${STAGES_FAILURES[$i]} -eq 0 ]]; then + TABLE="${TABLE}" + else + TABLE="${TABLE}" + fi + TABLE="${TABLE}" + done + TABLE="${TABLE}
Stage ✅ Passing ❌ Failure Details
${STAGES_NAMES[$i]}$((100-(${STAGES_FAILURES[$i]}*100/$N)))%
${STAGES_FAILURES_LOGS[$i]}
" + + git clone -b master --depth=1 https://github.com/commaai/ci-artifacts + cd ci-artifacts + git checkout -b "jenkins_test_report" + echo "$TABLE" >> jenkins_report + git add jenkins_report + git commit -m "jenkins report" + git remote set-url origin https://${env.CI_ARTIFACTS_TOKEN}@github.com/commaai/ci-artifacts.git + git push -f origin jenkins_test_report ''' currentBuild.result = 'SUCCESS'