Allow for Instanbul coverage with Playwright

.changeset/tidy-knives-agree.md

@@ -0,0 +1,5 @@
+---
+'@web/test-runner-playwright': patch
+---
+
+Support browser generated coverage without native instrumentation

integration/test-runner/tests/basic/coverage.test.js

@@ -0,0 +1,18 @@
+import { expect } from '../../../../../node_modules/@esm-bundle/chai/esm/chai.js';
+
+describe('basic test', () => {
+  it('works', () => {
+    window.__coverage__ = {
+      basic: {
+        path: 'basic.js',
+        statementMap: {},
+        fnMap: {},
+        branchMap: {},
+        s: {},
+        f: {},
+        b: {},
+      },
+    };
+    expect(true).to.equal(true);
+  });
+});

integration/test-runner/tests/basic/runBasicTest.ts

@@ -11,82 +11,115 @@ export function runBasicTest(
     const browserCount = config.browsers.length;
     let allSessions: TestSession[];
 
-    before(async () => {
-      const result = await runTests({
-        ...config,
-        files: [...(config.files ?? []), resolve(__dirname, 'browser-tests', '*.test.js')],
-        plugins: [...(config.plugins ?? []), legacyPlugin()],
-      });
-      allSessions = result.sessions;
+    describe('with coverage', () => {
+      it('passes coverage test', async () => {
+        const result = await runTests({
+          ...config,
+          coverage: true,
+          coverageConfig: {
+            nativeInstrumentation: false,
+          },
+          files: [...(config.files ?? []), resolve(__dirname, 'coverage.test.js')],
+          plugins: [...(config.plugins ?? []), legacyPlugin()],
+        });
+        const allSessions = result.sessions;
 
-      expect(allSessions.every(s => s.passed)).to.equal(true, 'All sessions should have passed');
-    });
+        expect(allSessions.every(s => s.passed)).to.equal(true, 'All sessions should have passed');
 
-    it('passes basic test', () => {
-      const sessions = allSessions.filter(s => s.testFile.endsWith('basic.test.js'));
-      expect(sessions.length === browserCount).to.equal(
-        true,
-        'Each browser should run basic.test.js',
-      );
-      for (const session of sessions) {
-        expect(session.testResults!.tests.length).to.equal(0);
-        expect(session.testResults!.suites.length).to.equal(1);
-        expect(session.testResults!.suites[0].tests.length).to.equal(1);
-        expect(session.testResults!.suites[0].tests.map(t => t.name)).to.eql(['works']);
-      }
+        const sessions = allSessions.filter(s => s.testFile.endsWith('coverage.test.js'));
+        expect(sessions.length === browserCount).to.equal(
+          true,
+          'Each browser should run coverage.test.js',
+        );
+        for (const session of sessions) {
+          expect(session.testCoverage).not.to.be.undefined;
+          expect(session.testCoverage!['basic']['path']).to.equal('basic.js');
+          expect(session.testResults!.tests.length).to.equal(0);
+          expect(session.testResults!.suites.length).to.equal(1);
+          expect(session.testResults!.suites[0].tests.length).to.equal(1);
+          expect(session.testResults!.suites[0].tests.map(t => t.name)).to.eql(['works']);
+        }
+      });
     });
 
-    it('passes js-syntax test', () => {
-      const sessions = allSessions.filter(s => s.testFile.endsWith('js-syntax.test.js'));
-      expect(sessions.length === browserCount).to.equal(
-        true,
-        'Each browser should run js-syntax.test.js',
-      );
-      for (const session of sessions) {
-        expect(session.testResults!.tests.map(t => t.name)).to.eql([
-          'supports object spread',
-          'supports async functions',
-          'supports exponentiation',
-          'supports classes',
-          'supports template literals',
-          'supports optional chaining',
-          'supports nullish coalescing',
-        ]);
-      }
-    });
+    describe('without coverage', () => {
+      before(async () => {
+        const result = await runTests({
+          ...config,
+          files: [...(config.files ?? []), resolve(__dirname, 'browser-tests', '*.test.js')],
+          plugins: [...(config.plugins ?? []), legacyPlugin()],
+        });
+        allSessions = result.sessions;
 
-    it('passes module-features test', () => {
-      const sessions = allSessions.filter(s => s.testFile.endsWith('module-features.test.js'));
-      expect(sessions.length === browserCount).to.equal(
-        true,
-        'Each browser should run module-features.test.js',
-      );
-      for (const session of sessions) {
-        expect(session.testResults!.tests.map(t => t.name)).to.eql([
-          'supports static imports',
-          'supports dynamic imports',
-          'supports import meta',
-        ]);
-      }
-    });
+        expect(allSessions.every(s => s.passed)).to.equal(true, 'All sessions should have passed');
+      });
 
-    it('passes timers test', () => {
-      const sessions = allSessions.filter(s => s.testFile.endsWith('timers.test.js'));
-      expect(sessions.length === browserCount).to.equal(
-        true,
-        'Each browser should run timers.test.js',
-      );
-      for (const session of sessions) {
-        expect(session.testResults!.tests.length).to.equal(0);
-        expect(session.testResults!.suites.length).to.equal(1);
-        expect(session.testResults!.suites[0].tests.map(t => t.name)).to.eql([
-          'can call setTimeout',
-          'can cancel setTimeout',
-          'can call and cancel setInterval',
-          'can call requestAnimationFrame',
-          'can cancel requestAnimationFrame',
-        ]);
-      }
+      it('passes basic test', () => {
+        const sessions = allSessions.filter(s => s.testFile.endsWith('basic.test.js'));
+        expect(sessions.length === browserCount).to.equal(
+          true,
+          'Each browser should run basic.test.js',
+        );
+        for (const session of sessions) {
+          expect(session.testResults!.tests.length).to.equal(0);
+          expect(session.testResults!.suites.length).to.equal(1);
+          expect(session.testResults!.suites[0].tests.length).to.equal(1);
+          expect(session.testResults!.suites[0].tests.map(t => t.name)).to.eql(['works']);
+        }
+      });
+
+      it('passes js-syntax test', () => {
+        const sessions = allSessions.filter(s => s.testFile.endsWith('js-syntax.test.js'));
+        expect(sessions.length === browserCount).to.equal(
+          true,
+          'Each browser should run js-syntax.test.js',
+        );
+        for (const session of sessions) {
+          expect(session.testResults!.tests.map(t => t.name)).to.eql([
+            'supports object spread',
+            'supports async functions',
+            'supports exponentiation',
+            'supports classes',
+            'supports template literals',
+            'supports optional chaining',
+            'supports nullish coalescing',
+          ]);
+        }
+      });
+
+      it('passes module-features test', () => {
+        const sessions = allSessions.filter(s => s.testFile.endsWith('module-features.test.js'));
+        expect(sessions.length === browserCount).to.equal(
+          true,
+          'Each browser should run module-features.test.js',
+        );
+        for (const session of sessions) {
+          expect(session.testResults!.tests.map(t => t.name)).to.eql([
+            'supports static imports',
+            'supports dynamic imports',
+            'supports import meta',
+          ]);
+        }
+      });
+
+      it('passes timers test', () => {
+        const sessions = allSessions.filter(s => s.testFile.endsWith('timers.test.js'));
+        expect(sessions.length === browserCount).to.equal(
+          true,
+          'Each browser should run timers.test.js',
+        );
+        for (const session of sessions) {
+          expect(session.testResults!.tests.length).to.equal(0);
+          expect(session.testResults!.suites.length).to.equal(1);
+          expect(session.testResults!.suites[0].tests.map(t => t.name)).to.eql([
+            'can call setTimeout',
+            'can cancel setTimeout',
+            'can call and cancel setInterval',
+            'can call requestAnimationFrame',
+            'can cancel requestAnimationFrame',
+          ]);
+        }
+      });
     });
   });
 }

packages/test-runner-playwright/src/PlaywrightLauncherPage.ts

@@ -44,9 +44,7 @@ export class PlaywrightLauncherPage {
   }
 
   async stopSession(): Promise<SessionResult> {
-    const testCoverage = this.nativeInstrumentationEnabledOnPage
-      ? await this.collectTestCoverage(this.config, this.testFiles)
-      : undefined;
+    const testCoverage = await this.collectTestCoverage(this.config, this.testFiles);
 
     // navigate to an empty page to kill any running code on the page, stopping timers and
     // breaking a potential endless reload loop
@@ -82,6 +80,10 @@ export class PlaywrightLauncherPage {
       );
     }
 
+    if (!this.nativeInstrumentationEnabledOnPage) {
+      return undefined;
+    }
+
     // get native coverage from playwright
     let coverage: V8Coverage[];
     if (this.product === 'chromium') {