From 0a781a622ee17a36d63320862c13a319cb7ff25b Mon Sep 17 00:00:00 2001 From: Riccardo Cipolleschi Date: Fri, 13 Dec 2024 14:31:13 +0000 Subject: [PATCH] Improve E2E testing for Android by retrying single tests --- .github/actions/maestro-android/action.yml | 2 +- .github/actions/maestro-ios/action.yml | 2 +- .github/workflow-scripts/maestro-android.js | 81 +++++++++++++++++++-- .github/workflows/test-all.yml | 6 +- packages/rn-tester/.maestro/flatlist.yml | 28 +++---- packages/rn-tester/.maestro/text.yml | 4 +- 6 files changed, 96 insertions(+), 27 deletions(-) diff --git a/.github/actions/maestro-android/action.yml b/.github/actions/maestro-android/action.yml index 99a8c0733098af..38770cf27c58d0 100644 --- a/.github/actions/maestro-android/action.yml +++ b/.github/actions/maestro-android/action.yml @@ -35,7 +35,7 @@ runs: steps: - name: Installing Maestro shell: bash - run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash + run: export MAESTRO_VERSION=1.39.4; curl -Ls "https://get.maestro.mobile.dev" | bash - name: Set up JDK 17 if: ${{ inputs.install-java == 'true' }} uses: actions/setup-java@v4 diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index e5dd927bec32ed..ea0a08ccca2b91 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -31,7 +31,7 @@ runs: steps: - name: Installing Maestro shell: bash - run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash + run: export MAESTRO_VERSION=1.39.4; curl -Ls "https://get.maestro.mobile.dev" | bash - name: Installing Maestro dependencies shell: bash run: | diff --git a/.github/workflow-scripts/maestro-android.js b/.github/workflow-scripts/maestro-android.js index f6533bd0933cdc..293187622ecc91 100644 --- a/.github/workflow-scripts/maestro-android.js +++ b/.github/workflow-scripts/maestro-android.js @@ -8,6 +8,7 @@ */ const childProcess = require('child_process'); +const fs = require('fs'); const usage = ` === Usage === @@ -33,6 +34,69 @@ const MAESTRO_FLOW = args[2]; const IS_DEBUG = args[3] === 'debug'; const WORKING_DIRECTORY = args[4]; +const MAX_RETRIES = 3; + +async function rebootDevice() { + console.info('Android device hanged, rebooting...'); + childProcess.execSync('adb -e reboot && adb wait-for-device', { + stdio: 'inherit', + }); + let deviceReady = String( + childProcess.execSync(`adb shell getprop dev.bootcomplete`), + ).trim(); + while (!deviceReady) { + await sleep(250); + deviceReady = String( + childProcess.execSync(`adb shell getprop dev.bootcomplete`), + ).trim(); + } +} + +async function executeFlowWithRetries(maestroFlow) { + let errors = []; + for (let currentAttempt = 0; currentAttempt < MAX_RETRIES; currentAttempt++) { + try { + await rebootDevice(); + childProcess.execSync( + `MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${maestroFlow} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`, + {stdio: 'inherit'}, + ); + return; + } catch (err) { + errors.push(err); + if (currentAttempt < MAX_RETRIES) { + console.info(`Retrying ${maestroFlow}...`); + } else { + throw new AggregateError(errors); + } + } + } +} + +async function executeFlowsInFolder(folder) { + const files = fs.readdirSync(folder); + let errors = []; + for (const file of files) { + const filePath = `${folder}/${file}`; + if (fs.statSync(filePath).isDirectory()) { + try { + await executeFlowsInFolder(filePath); + } catch (err) { + errors.concat(err.errors); + } + } else if (filePath.endsWith('.yml')) { + try { + await executeFlowWithRetries(filePath); + } catch (err) { + errors.push(err); + } + } + } + if (errors.length > 0) { + throw new AggregateError(errors); + } +} + async function main() { console.info('\n=============================='); console.info('Running tests for Android with the following parameters:'); @@ -57,10 +121,10 @@ async function main() { }); metroProcess.unref(); console.info(`- Metro PID: ${metroProcess.pid}`); - } - console.info('Wait For Metro to Start'); - await sleep(5000); + console.info('Wait For Metro to Start'); + await sleep(5000); + } console.info('Start the app'); childProcess.execSync(`adb shell monkey -p ${APP_ID} 1`, {stdio: 'ignore'}); @@ -76,10 +140,13 @@ async function main() { console.info(`Start testing ${MAESTRO_FLOW}`); let error = null; try { - childProcess.execSync( - `MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${MAESTRO_FLOW} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`, - {stdio: 'inherit'}, - ); + if (MAESTRO_FLOW.endsWith('.yml')) { + // Single flow + await executeFlowWithRetries(MAESTRO_FLOW); + } else { + console.info(`Multiple flows in ${MAESTRO_FLOW}`); + await executeFlowsInFolder(MAESTRO_FLOW); + } } catch (err) { error = err; } finally { diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 317fdbc49f4fec..00cbc4b26222ab 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -432,13 +432,13 @@ jobs: uses: ./.github/actions/build-android with: release-type: ${{ needs.set_release_type.outputs.RELEASE_TYPE }} - run-e2e-tests: ${{ github.ref == 'refs/heads/main' || contains(github.ref, 'stable') || inputs.run-e2e-tests }} + run-e2e-tests: true #${{ github.ref == 'refs/heads/main' || contains(github.ref, 'stable') || inputs.run-e2e-tests }} test_e2e_android_rntester: # Temporarily disable RNTester tests on Android as they are quite flaky and they make CI always red # if: ${{ github.ref == 'refs/heads/main' || contains(github.ref, 'stable') || inputs.run-e2e-tests }} - if: ${{ contains(github.ref, 'stable') || inputs.run-e2e-tests }} - runs-on: ubuntu-latest + # if: ${{ contains(github.ref, 'stable') || inputs.run-e2e-tests }} + runs-on: 4-core-ubuntu needs: [build_android] strategy: fail-fast: false diff --git a/packages/rn-tester/.maestro/flatlist.yml b/packages/rn-tester/.maestro/flatlist.yml index 6db8d028cebf2b..73a59a789f0ac8 100644 --- a/packages/rn-tester/.maestro/flatlist.yml +++ b/packages/rn-tester/.maestro/flatlist.yml @@ -11,20 +11,20 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo id: "Flatlist" - tapOn: id: "Basic" -# - assertVisible: -# id: "item_550" -# - repeat: -# while: -# notVisible: -# id: "item_600" # should trigger a reload -# commands: -# - swipe: -# start: 50%, 85% -# end: 50%, 50% -# - waitForAnimationToEnd: # wait for the reload to happen -# timeout: 1000 -# - assertVisible: -# id: "item_600" +- assertVisible: + id: "item_550" +- repeat: + while: + notVisible: + id: "item_600" # should trigger a reload + commands: + - swipe: + start: 50%, 85% + end: 50%, 50% + - waitForAnimationToEnd: # wait for the reload to happen + timeout: 1000 +- assertVisible: + id: "item_600" - assertVisible: text: "Empty:" - tapOn: diff --git a/packages/rn-tester/.maestro/text.yml b/packages/rn-tester/.maestro/text.yml index db74042e66bf0a..61fd86d7a77ea9 100644 --- a/packages/rn-tester/.maestro/text.yml +++ b/packages/rn-tester/.maestro/text.yml @@ -6,7 +6,8 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo element: id: "Text" direction: DOWN - speed: 60 + speed: 50 + timeout: 60000 - tapOn: id: "Text" - scrollUntilVisible: @@ -15,6 +16,7 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo direction: DOWN speed: 10 visibilityPercentage: 100 + timeout: 60000 - assertVisible: "Text with background color only" - assertVisible: "Text with background color and uniform borderRadii" - assertVisible: "Text with background color and non-uniform borders"