From 35a27c8603ac961a3cf5f2cf384c4a9d40cca0d9 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 1 Aug 2023 16:33:11 -0700 Subject: [PATCH] test(full-page-screenshot): add node verification tests and debug tool --- cli/test/fixtures/screenshot-nodes.html | 58 +++ core/gather/gatherers/full-page-screenshot.js | 4 +- core/scripts/full-page-screenshot-debug.js | 80 +++++ .../gatherers/full-page-screenshot-test.js | 338 ++++++++++++++++++ types/lhr/lhr.d.ts | 2 +- 5 files changed, 479 insertions(+), 3 deletions(-) create mode 100644 cli/test/fixtures/screenshot-nodes.html create mode 100644 core/scripts/full-page-screenshot-debug.js diff --git a/cli/test/fixtures/screenshot-nodes.html b/cli/test/fixtures/screenshot-nodes.html new file mode 100644 index 000000000000..a90622f0d113 --- /dev/null +++ b/cli/test/fixtures/screenshot-nodes.html @@ -0,0 +1,58 @@ + + + + + + + + +

Screenshot tester 2

+ + + +
+
+ +
+ diff --git a/core/gather/gatherers/full-page-screenshot.js b/core/gather/gatherers/full-page-screenshot.js index 975b89bb24ca..b852075e03dc 100644 --- a/core/gather/gatherers/full-page-screenshot.js +++ b/core/gather/gatherers/full-page-screenshot.js @@ -14,7 +14,7 @@ import {waitForNetworkIdle} from '../driver/wait-for-condition.js'; // JPEG quality setting // Exploration and examples of reports using different quality settings: https://docs.google.com/document/d/1ZSffucIca9XDW2eEwfoevrk-OTl7WQFeMf0CgeJAA8M/edit# // Note: this analysis was done for JPEG, but now we use WEBP. -const FULL_PAGE_SCREENSHOT_QUALITY = 30; +const FULL_PAGE_SCREENSHOT_QUALITY = process.env.LH_FPS_TEST ? 30 : 100; // https://developers.google.com/speed/webp/faq#what_is_the_maximum_size_a_webp_image_can_be const MAX_WEBP_SIZE = 16383; @@ -160,7 +160,7 @@ class FullPageScreenshot extends BaseGatherer { for (const [node, id] of lhIdToElements.entries()) { // @ts-expect-error - getBoundingClientRect put into scope via stringification const rect = getBoundingClientRect(node); - nodes[id] = rect; + nodes[id] = {id: node.id, ...rect}; } return nodes; diff --git a/core/scripts/full-page-screenshot-debug.js b/core/scripts/full-page-screenshot-debug.js new file mode 100644 index 000000000000..9daa9d711e6c --- /dev/null +++ b/core/scripts/full-page-screenshot-debug.js @@ -0,0 +1,80 @@ +/** + * @license Copyright 2023 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +// node core/scripts/full-page-screenshot-debug.js latest-run/lhr.report.json | xargs "$CHROME_PATH" + +import * as fs from 'fs'; + +import * as puppeteer from 'puppeteer-core'; +import {getChromePath} from 'chrome-launcher'; + +/** + * @param {LH.Result} lhr + * @return {Promise} + */ +async function getDebugImage(lhr) { + if (!lhr.fullPageScreenshot) { + return ''; + } + + const browser = await puppeteer.launch({ + headless: true, + executablePath: getChromePath(), + ignoreDefaultArgs: ['--enable-automation'], + }); + const page = await browser.newPage(); + + const debugDataUrl = await page.evaluate(async (fullPageScreenshot) => { + const img = await new Promise((resolve, reject) => { + // eslint-disable-next-line no-undef + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = fullPageScreenshot.screenshot.data; + }); + + // eslint-disable-next-line no-undef + const canvasEl = document.createElement('canvas'); + canvasEl.width = img.width; + canvasEl.height = img.height; + const ctx = canvasEl.getContext('2d'); + if (!ctx) return ''; + + ctx.drawImage(img, 0, 0); + for (const [lhId, node] of Object.entries(fullPageScreenshot.nodes)) { + if (!node.width && !node.height) continue; + + ctx.strokeStyle = '#D3E156'; + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.rect(node.left, node.top, node.width, node.height); + ctx.stroke(); + + const txt = node.id || lhId; + const txtWidth = Math.min(ctx.measureText(txt).width, node.width); + const txtHeight = 10; + const txtTop = node.top - 3; + const txtLeft = node.left; + ctx.fillStyle = '#FFFFFF88'; + ctx.fillRect(txtLeft, txtTop, txtWidth, txtHeight); + ctx.fillStyle = '#000000'; + ctx.lineWidth = 1; + ctx.textBaseline = 'top'; + ctx.fillText(txt, txtLeft, txtTop); + } + + return canvasEl.toDataURL(); + }, lhr.fullPageScreenshot); + + await browser.close(); + + return debugDataUrl; +} + +const lhr = JSON.parse(fs.readFileSync(process.argv[2], 'utf-8')); +const result = await getDebugImage(lhr); + +console.log(result); diff --git a/core/test/gather/gatherers/full-page-screenshot-test.js b/core/test/gather/gatherers/full-page-screenshot-test.js index beb9864f6b7c..8230034573bc 100644 --- a/core/test/gather/gatherers/full-page-screenshot-test.js +++ b/core/test/gather/gatherers/full-page-screenshot-test.js @@ -4,8 +4,14 @@ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +import * as puppeteer from 'puppeteer-core'; +import {getChromePath} from 'chrome-launcher'; + +import * as LH from '../../../../types/lh.js'; import {createMockContext} from '../../gather/mock-driver.js'; import FullPageScreenshotGatherer from '../../../gather/gatherers/full-page-screenshot.js'; +import lighthouse from '../../../index.js'; +import {Server} from '../../../../cli/test/fixtures/static-server.js'; /** @type {{width: number, height: number}} */ let contentSize; @@ -202,4 +208,336 @@ describe('FullPageScreenshot gatherer', () => { } ); }); + + // Tests that our node rects line up with content in the screenshot image data. + // This uses "screenshot-nodes.html", which has elements of solid colors to make verification simple. + // To verify a node rect is correct, each pixel in its area is looked at in the screenshot data, and is checked + // for the expected color. Due to compression artifacts, there are thresholds involved instead of exact matches. + describe('end-to-end integration test', () => { + const port = 10503; + let serverBaseUrl = '' + /** @type {StaticServer} */; + let server; + /** @type {puppeteer.Browser} */ + let browser; + /** @type {puppeteer.Page} */ + let page; + + before(async () => { + browser = await puppeteer.launch({ + headless: true, + executablePath: getChromePath(), + ignoreDefaultArgs: ['--enable-automation'], + }); + + server = new Server(port); + await server.listen(port, '127.0.0.1'); + serverBaseUrl = `http://localhost:${server.getPort()}`; + + // Tell gatherer to use 100 quality. + process.env.LH_FPS_TEST = '1'; + }); + + after(async () => { + await browser.close(); + await server.close(); + }); + + beforeEach(async () => { + page = await browser.newPage(); + }); + + afterEach(async () => { + await page.close(); + }); + + /** + * @typedef NodeAnalysisResult + * @property {string} id + * @property {boolean} success + * @property {number[][]} debugData + */ + + /** + * @param {puppeteer.Page} page + * @param {LH.Result} lhr + * @param {DebugFormat} debugFormat + * @return {Promise} + */ + function analyzeScreenshotNodes(page, lhr, debugFormat) { + const options = { + fullPageScreenshot: lhr.fullPageScreenshot, + // We currently can't get lossless encodings here, so we must allow for some difference. + // I found 2.4 is what I get with the test data for regions that match, and 100+ for regions that are clearly not matching. + // So using 5 seems a good bet. + // If we could get lossless webp/png here, this should go away. + // TODO https://bugs.chromium.org/p/chromium/issues/detail?id=1469183 + individualPixelColorThreshold: 5, + debugFormat, + }; + + return page.evaluate(async (options) => { + function hexToRgb(hex) { + const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); + return { + r: parseInt(result[1], 16), + g: parseInt(result[2], 16), + b: parseInt(result[3], 16), + }; + } + + // https://www.compuphase.com/cmetric.htm + function colorDistance(e1, e2) { + const rmean = (e1.r + e2.r) / 2; + const r = e1.r - e2.r; + const g = e1.g - e2.g; + const b = e1.b - e2.b; + return Math.sqrt( + (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8)); + } + + const img = await new Promise((resolve, reject) => { + // eslint-disable-next-line no-undef + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = options.fullPageScreenshot.screenshot.data; + }); + + // eslint-disable-next-line no-undef + const canvasEl = document.createElement('canvas'); + canvasEl.width = img.width; + canvasEl.height = img.height; + const ctx = canvasEl.getContext('2d'); + ctx.drawImage(img, 0, 0); + + const results = []; + for (const node of Object.values(options.fullPageScreenshot.nodes)) { + if (!(node.id.includes('green') || node.id.includes('red'))) { + continue; + } + + const expectedColor = hexToRgb(node.id.includes('green') ? '#427f36' : '#8a4343'); + const result = {id: node.id, success: true}; + results.push(result); + + const right = Math.min(node.right, canvasEl.width - 1); + const bottom = Math.min(node.bottom, canvasEl.height - 1); + + let pixelDifferCount = 0; + const debugData = []; + for (let y = node.top; y <= bottom; y++) { + const row = []; + debugData.push(row); + + for (let x = node.left; x <= right; x++) { + const [r, g, b] = ctx.getImageData(x, y, 1, 1).data; + const delta = colorDistance(expectedColor, {r, g, b}); + const passesThreshold = delta < options.individualPixelColorThreshold; + if (!passesThreshold) pixelDifferCount += 1; + + if (options.debugFormat === 'color') { + row.push((r << 16) | (g << 8) | b); + } else if (options.debugFormat === 'delta') { + row.push(delta); + } else if (options.debugFormat === 'threshold') { + row.push(passesThreshold); + } + } + } + + // Some amount of pixels along the borders will differ, either because of compression artifacts or good old-fashioned + // off-by-one issues. Looking at the 'threshold' debug visualization, it seems we can count on these pixels differing: + // * first two rows + // * last three rows + // * first column + // * last two columns + // The following value takes this into account. Much more pixels than this differing, and we've likely got a visual mismatch. + // It's an overcount, but the math is simpler and we want some additional leeway anyhow. + const differingPixelCountThreshold = node.width * 5 + node.height * 3; + result.success = pixelDifferCount < differingPixelCountThreshold; + + if (options.debugFormat) result.debugData = debugData; + } + + return results; + }, options); + } + + /** + * @param {NodeAnalysisResult[]} results + * @param {DebugFormat} debugFormat + */ + function visualizeDebugData(results, debugFormat) { + for (const result of results) { + console.log(`\n=== ${result.id} ${result.success ? 'success' : 'failure'} ===\n`); + + const columns = result.debugData; + for (let y = 0; y < columns.length; y++) { + let line = ''; + for (let x = 0; x < columns[0].length; x++) { + if (debugFormat === 'color') { + line += columns[y][x].toString(16).padStart(6, '0') + ' '; + } else if (debugFormat === 'delta') { + line += columns[y][x].toFixed(1) + ' '; + } else if (debugFormat === 'threshold') { + line += columns[y][x] ? 'O' : 'X'; + } + } + console.log(line); + } + } + } + + /** + * @param {LH.Result} lhr + * @param {Array} rectExpectations + */ + async function verifyNodeRectsAlignWithImage(lhr, rectExpectations) { + if (!lhr.fullPageScreenshot) throw new Error('no screenshot'); + + // First check we recieved all the expected nodes. + const nodes = Object.values(lhr.fullPageScreenshot.nodes); + for (const expectation of rectExpectations) { + const nodeSeen = nodes.find(node => node.id === expectation.id); + if (!nodeSeen) throw new Error(`did not find node for id ${expectation.id}`); + + const {id, left, top, right, bottom} = nodeSeen; + expect({id, left, top, right, bottom}).toEqual(expectation); + } + + // Now check that the image contents line up with what we think the nodes are. + + /** @type {DebugFormat} */ + const debugFormat = process.env.LH_FPS_DEBUG ?? false; + const results = await analyzeScreenshotNodes(page, lhr, debugFormat); + + // Very helpful for debugging. Set env LH_FPS_DEBUG to one of the valid debug formats. + if (debugFormat) { + console.log(lhr.fullPageScreenshot.screenshot.data); + visualizeDebugData(results, debugFormat); + } + + const failingIds = results.filter(r => !r.success).map(r => r.id); + expect(failingIds).toEqual([]); + + expect(results.length).toBeGreaterThan(0); + } + + it('mobile dpr 1', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 43, + bottom: 53, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 93, + bottom: 153, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('mobile dpr 1 tiny viewport', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 61, + bottom: 71, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 111, + bottom: 171, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 100, height: 200, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('mobile dpr 1.75', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 43, + bottom: 53, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 93, + bottom: 153, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.75}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('desktop', async () => { + const rectExpectations = [ + { + id: 'el-1-red', + top: 43, + bottom: 53, + left: 18, + right: 178, + }, + { + id: 'el-2-green', + top: 93, + bottom: 153, + left: 48, + right: 108, + }, + ]; + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: false, width: 1350, height: 940, deviceScaleFactor: 1}, + formFactor: 'desktop', + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + await verifyNodeRectsAlignWithImage(runnerResult.lhr, rectExpectations); + }); + + it('grow', async () => { + const runnerResult = await lighthouse(`${serverBaseUrl}/screenshot-nodes.html?grow`, { + onlyAudits: ['largest-contentful-paint-element', 'aria-required-parent'], + screenEmulation: {mobile: true, width: 600, height: 900, deviceScaleFactor: 1.0}, + }, undefined, page); + if (!runnerResult) throw new Error('no runner result'); + + // No rect expectations, because we can't know what they'll be ahead of time. + await verifyNodeRectsAlignWithImage(runnerResult.lhr, []); + }); + }); }); diff --git a/types/lhr/lhr.d.ts b/types/lhr/lhr.d.ts index 3e401eb244eb..b3320f6ffe2e 100644 --- a/types/lhr/lhr.d.ts +++ b/types/lhr/lhr.d.ts @@ -150,7 +150,7 @@ declare module Result { width: number; height: number; }; - nodes: Record; + nodes: Record; } /**