Improve E2E testing for Android by retrying single tests

.github/actions/maestro-android/action.yml

@@ -35,7 +35,7 @@ runs:
   steps:
     - name: Installing Maestro
       shell: bash
-      run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
+      run: export MAESTRO_VERSION=1.39.4; curl -Ls "https://get.maestro.mobile.dev" | bash
     - name: Set up JDK 17
       if: ${{ inputs.install-java == 'true' }}
       uses: actions/setup-java@v4

.github/actions/maestro-ios/action.yml

@@ -31,7 +31,7 @@ runs:
   steps:
     - name: Installing Maestro
       shell: bash
-      run: export MAESTRO_VERSION=1.36.0; curl -Ls "https://get.maestro.mobile.dev" | bash
+      run: export MAESTRO_VERSION=1.39.4; curl -Ls "https://get.maestro.mobile.dev" | bash
     - name: Installing Maestro dependencies
       shell: bash
       run: |

.github/workflow-scripts/maestro-android.js

@@ -8,6 +8,7 @@
  */
 
 const childProcess = require('child_process');
+const fs = require('fs');
 
 const usage = `
 === Usage ===
@@ -33,6 +34,69 @@ const MAESTRO_FLOW = args[2];
 const IS_DEBUG = args[3] === 'debug';
 const WORKING_DIRECTORY = args[4];
 
+const MAX_RETRIES = 3;
+
+async function rebootDevice() {
+  console.info('Android device hanged, rebooting...');
+  childProcess.execSync('adb -e reboot && adb wait-for-device', {
+    stdio: 'inherit',
+  });
+  let deviceReady = String(
+    childProcess.execSync(`adb shell getprop dev.bootcomplete`),
+  ).trim();
+  while (!deviceReady) {
+    await sleep(250);
+    deviceReady = String(
+      childProcess.execSync(`adb shell getprop dev.bootcomplete`),
+    ).trim();
+  }
+}
+
+async function executeFlowWithRetries(maestroFlow) {
+  let errors = [];
+  for (let currentAttempt = 0; currentAttempt < MAX_RETRIES; currentAttempt++) {
+    try {
+      await rebootDevice();
+      childProcess.execSync(
+        `MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${maestroFlow} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`,
+        {stdio: 'inherit'},
+      );
+      return;
+    } catch (err) {
+      errors.push(err);
+      if (currentAttempt < MAX_RETRIES) {
+        console.info(`Retrying ${maestroFlow}...`);
+      } else {
+        throw new AggregateError(errors);
+      }
+    }
+  }
+}
+
+async function executeFlowsInFolder(folder) {
+  const files = fs.readdirSync(folder);
+  let errors = [];
+  for (const file of files) {
+    const filePath = `${folder}/${file}`;
+    if (fs.statSync(filePath).isDirectory()) {
+      try {
+        await executeFlowsInFolder(filePath);
+      } catch (err) {
+        errors.concat(err.errors);
+      }
+    } else if (filePath.endsWith('.yml')) {
+      try {
+        await executeFlowWithRetries(filePath);
+      } catch (err) {
+        errors.push(err);
+      }
+    }
+  }
+  if (errors.length > 0) {
+    throw new AggregateError(errors);
+  }
+}
+
 async function main() {
   console.info('\n==============================');
   console.info('Running tests for Android with the following parameters:');
@@ -57,10 +121,10 @@ async function main() {
     });
     metroProcess.unref();
     console.info(`- Metro PID: ${metroProcess.pid}`);
-  }
 
-  console.info('Wait For Metro to Start');
-  await sleep(5000);
+    console.info('Wait For Metro to Start');
+    await sleep(5000);
+  }
 
   console.info('Start the app');
   childProcess.execSync(`adb shell monkey -p ${APP_ID} 1`, {stdio: 'ignore'});
@@ -76,10 +140,13 @@ async function main() {
   console.info(`Start testing ${MAESTRO_FLOW}`);
   let error = null;
   try {
-    childProcess.execSync(
-      `MAESTRO_DRIVER_STARTUP_TIMEOUT=120000 $HOME/.maestro/bin/maestro test ${MAESTRO_FLOW} --format junit -e APP_ID=${APP_ID} --debug-output /tmp/MaestroLogs`,
-      {stdio: 'inherit'},
-    );
+    if (MAESTRO_FLOW.endsWith('.yml')) {
+      // Single flow
+      await executeFlowWithRetries(MAESTRO_FLOW);
+    } else {
+      console.info(`Multiple flows in ${MAESTRO_FLOW}`);
+      await executeFlowsInFolder(MAESTRO_FLOW);
+    }
   } catch (err) {
     error = err;
   } finally {

.github/workflows/test-all.yml

@@ -432,13 +432,13 @@ jobs:
         uses: ./.github/actions/build-android
         with:
           release-type: ${{ needs.set_release_type.outputs.RELEASE_TYPE }}
-          run-e2e-tests: ${{ github.ref == 'refs/heads/main' || contains(github.ref,  'stable') || inputs.run-e2e-tests }}
+          run-e2e-tests: true #${{ github.ref == 'refs/heads/main' || contains(github.ref,  'stable') || inputs.run-e2e-tests }}
 
   test_e2e_android_rntester:
     # Temporarily disable RNTester tests on Android as they are quite flaky and they make CI always red
     # if: ${{ github.ref == 'refs/heads/main' || contains(github.ref,  'stable') || inputs.run-e2e-tests }}
-    if: ${{ contains(github.ref,  'stable') || inputs.run-e2e-tests }}
-    runs-on: ubuntu-latest
+    # if: ${{ contains(github.ref,  'stable') || inputs.run-e2e-tests }}
+    runs-on: 4-core-ubuntu
     needs: [build_android]
     strategy:
       fail-fast: false

packages/rn-tester/.maestro/flatlist.yml

@@ -11,20 +11,20 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo
     id: "Flatlist"
 - tapOn:
     id: "Basic"
-# - assertVisible:
-#     id: "item_550"
-# - repeat:
-#     while:
-#       notVisible:
-#         id: "item_600" # should trigger a reload
-#     commands:
-#       - swipe:
-#           start: 50%, 85%
-#           end: 50%, 50%
-#       - waitForAnimationToEnd: # wait for the reload to happen
-#           timeout: 1000
-# - assertVisible:
-#     id: "item_600"
+- assertVisible:
+    id: "item_550"
+- repeat:
+    while:
+      notVisible:
+        id: "item_600" # should trigger a reload
+    commands:
+      - swipe:
+          start: 50%, 85%
+          end: 50%, 50%
+      - waitForAnimationToEnd: # wait for the reload to happen
+          timeout: 1000
+- assertVisible:
+    id: "item_600"
 - assertVisible:
     text: "Empty:"
 - tapOn:

packages/rn-tester/.maestro/text.yml

@@ -6,7 +6,8 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo
     element:
       id: "Text"
     direction: DOWN
-    speed: 60
+    speed: 50
+    timeout: 60000
 - tapOn:
     id: "Text"
 - scrollUntilVisible:
@@ -15,6 +16,7 @@ appId: ${APP_ID} # iOS: com.meta.RNTester.localDevelopment | Android: com.facebo
     direction: DOWN
     speed: 10
     visibilityPercentage: 100
+    timeout: 60000
 - assertVisible: "Text with background color only"
 - assertVisible: "Text with background color and uniform borderRadii"
 - assertVisible: "Text with background color and non-uniform borders"