tests(unit): retry failures and upload failure artifacts (#15378)

.github/workflows/unit.yml

@@ -84,6 +84,13 @@ jobs:
     - name: yarn test-viewer
       run: yarn build-viewer && xvfb-run --auto-servernum bash $GITHUB_WORKSPACE/.github/scripts/test-retry.sh yarn test-viewer
 
+    - name: Upload failures
+      if: failure()
+      uses: actions/upload-artifact@v1
+      with:
+        name: Unit (ubuntu; node ${{ matrix.node }})
+        path: .tmp/unit-failures/
+
   # For windows, just test the potentially platform-specific code.
   unit-windows:
     runs-on: windows-latest

core/test/scenarios/api-test-pptr.js

@@ -75,6 +75,7 @@ describe('Individual modes API', function() {
       if (!result) throw new Error('Lighthouse failed to produce a result');
 
       const {lhr, artifacts} = result;
+      state.saveTrace(artifacts.Trace);
       expect(artifacts.URL).toEqual({
         finalDisplayedUrl: `${state.serverBaseUrl}/onclick.html#done`,
       });
@@ -133,6 +134,8 @@ describe('Individual modes API', function() {
 
       if (!result) throw new Error('Lighthouse failed to produce a result');
 
+      state.saveTrace(result.artifacts.Trace);
+
       expect(result.artifacts.URL).toEqual({
         finalDisplayedUrl: `${serverBaseUrl}/onclick.html#done`,
       });
@@ -165,6 +168,8 @@ describe('Individual modes API', function() {
 
       if (!result) throw new Error('Lighthouse failed to produce a result');
 
+      state.saveTrace(result.artifacts.Trace);
+
       const networkRequestsDetails = /** @type {LH.Audit.Details.Table} */ (
         result.lhr.audits['network-requests'].details);
       const networkRequests = networkRequestsDetails?.items
@@ -222,6 +227,7 @@ Array [
       if (!result) throw new Error('Lighthouse failed to produce a result');
 
       const {lhr, artifacts} = result;
+      state.saveTrace(artifacts.Trace);
       expect(artifacts.URL).toEqual({
         requestedUrl: url,
         mainDocumentUrl: url,
@@ -263,6 +269,7 @@ Array [
       expect(requestor).toHaveBeenCalled();
 
       const {lhr, artifacts} = result;
+      state.saveTrace(artifacts.Trace);
       expect(lhr.requestedUrl).toEqual(requestedUrl);
       expect(lhr.finalDisplayedUrl).toEqual(mainDocumentUrl);
       expect(artifacts.URL).toEqual({

core/test/scenarios/pptr-test-utils.js

@@ -4,11 +4,14 @@
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
  */
 
+import fs from 'fs';
+
 import {before, beforeEach, after, afterEach} from 'mocha';
 import * as puppeteer from 'puppeteer-core';
 import {getChromePath} from 'chrome-launcher';
 
 import {Server} from '../../../cli/test/fixtures/static-server.js';
+import {LH_ROOT} from '../../../root.js';
 
 /** @typedef {InstanceType<typeof import('../../../cli/test/fixtures/static-server.js').Server>} StaticServer */
 
@@ -22,12 +25,17 @@ const FLAKY_AUDIT_IDS_APPLICABILITY = new Set([
   'layout-shift-elements', // Depends on if the JS takes too long after input to be ignored for layout shift.
 ]);
 
+const UNIT_OUTPUT_DIR = `${LH_ROOT}/.tmp/unit-failures`;
+
 function createTestState() {
   /** @param {string} name @return {any} */
   const any = name => new Proxy({}, {get: () => {
     throw new Error(`${name} used without invoking \`state.before\``);
   }});
 
+  /** @type {LH.Trace|undefined} */
+  let trace;
+
   return {
     browser: /** @type {puppeteer.Browser} */ (any('browser')),
     page: /** @type {puppeteer.Page} */ (any('page')),
@@ -68,17 +76,37 @@ function createTestState() {
       });
 
       beforeEach(async () => {
+        trace = undefined;
+        console.log('########');
         this.page = await this.browser.newPage();
       });
 
       afterEach(async () => {
         await this.page.close();
       });
 
+      afterEach(function() {
+        // eslint-disable-next-line no-invalid-this
+        const currentTest = this.currentTest;
+        if (currentTest?.state === 'failed' && trace) {
+          const dirname = currentTest.fullTitle().replace(/[^a-z0-9]/gi, '_').toLowerCase();
+          const testOutputDir = `${UNIT_OUTPUT_DIR}/${dirname}`;
+          fs.mkdirSync(testOutputDir, {recursive: true});
+          fs.writeFileSync(`${testOutputDir}/trace.json`, JSON.stringify(trace, null, 2));
+        }
+      });
+
       after(async () => {
         await this.browser.close();
       });
     },
+
+    /**
+     * @param {LH.Trace} testTrace
+     */
+    saveTrace(testTrace) {
+      trace = testTrace;
+    },
   };
 }
 

core/test/scenarios/start-end-navigation-test-pptr.js

@@ -37,6 +37,8 @@ describe('Start/End navigation', function() {
     const lhr = flowResult.steps[0].lhr;
     const artifacts = flowArtifacts.gatherSteps[0].artifacts;
 
+    state.saveTrace(artifacts.Trace);
+
     expect(artifacts.URL).toEqual({
       requestedUrl: `${state.serverBaseUrl}/?redirect=/index.html`,
       mainDocumentUrl: `${state.serverBaseUrl}/index.html`,

core/test/scripts/run-mocha-tests.js

@@ -146,6 +146,14 @@ const rawArgv = y
     'require': {
       type: 'string',
     },
+    'retries': {
+      type: 'number',
+      default: process.env.CI ? 5 : undefined,
+    },
+    'forbidOnly': {
+      type: 'boolean',
+      default: Boolean(process.env.CI),
+    },
   })
   .wrap(y.terminalWidth())
   .argv;
@@ -257,8 +265,10 @@ function exit({numberFailures, numberMochaInvocations}) {
  * @typedef OurMochaArgs
  * @property {RegExp | string | undefined} grep
  * @property {boolean} bail
+ * @property {boolean} forbidOnly
  * @property {boolean} parallel
  * @property {string | undefined} require
+ * @property {number | undefined} retries
  */
 
 /**
@@ -278,9 +288,11 @@ async function runMocha(tests, mochaArgs, invocationNumber) {
       timeout: 20_000,
       bail: mochaArgs.bail,
       grep: mochaArgs.grep,
+      forbidOnly: mochaArgs.forbidOnly,
       // TODO: not working
       // parallel: tests.length > 1 && mochaArgs.parallel,
       parallel: false,
+      retries: mochaArgs.retries,
     });
 
     // @ts-expect-error - not in types.
@@ -325,6 +337,8 @@ async function main() {
     bail: argv.bail,
     parallel: argv.parallel,
     require: argv.require,
+    retries: argv.retries,
+    forbidOnly: argv.forbidOnly,
   };
 
   mochaGlobalSetup();

package.json

@@ -56,7 +56,7 @@
     "unit-viewer": "yarn mocha --testMatch viewer/**/*-test.js",
     "unit-flow": "bash flow-report/test/run-flow-report-tests.sh",
     "unit": "yarn unit-flow && yarn mocha",
-    "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit --forbid-only",
+    "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit",
     "core-unit": "yarn unit-core",
     "cli-unit": "yarn unit-cli",
     "viewer-unit": "yarn unit-viewer",