Merge pull request #3436 from artilleryio/feat/playwright-tests-as-code

.github/workflows/examples.yml

@@ -12,6 +12,33 @@ env:
   CLI_NOTE: Running from the Official Artillery Github Action! 😀
 
 jobs:
+  browser-load-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    env:
+      CWD: ./examples/browser-load-testing-playwright
+    defaults:
+      run:
+        working-directory: ${{ env.CWD }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Run example
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20.x'
+      - name: Install dependencies
+        run: npm ci
+      - name: Run test
+        run: |
+          $ARTILLERY_BINARY_PATH run browser-load-test.ts --record --tags ${{ env.CLI_TAGS }},group:browser-load-test --note "${{ env.CLI_NOTE }}"
+        working-directory: ${{ env.CWD }}
+        env:
+          ARTILLERY_BINARY_PATH: ${{ env.ARTILLERY_BINARY_PATH }}
+          ARTILLERY_CLOUD_ENDPOINT: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
+          ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
+
   http-metrics-by-endpoint:
     runs-on: ubuntu-latest
     timeout-minutes: 10

examples/browser-load-testing-playwright/README.md

@@ -1,14 +1,24 @@
-# Load testing and smoke testing with real browsers
+# Load testing and smoke testing with headless browsers
 
+Artillery can run Playwright scripts as performance tests. This example shows you how to run a simple load test, a smoke test, and how to track custom metrics for part of the flow.
 
-Ever wished you could run load tests with *real browsers*? Well, now you can! You can combine Artillery with Playwright to run full browser tests, and this example shows you how. We can run both load tests, and smoke tests with headless browsers.
+- [Why load test with headless browsers?](https://www.artillery.io/docs/playwright#why-load-test-with-headless-browsers)
 
+> [!TIP]
+> Artillery's uses YAML as its default configuration format, but Playwright tests can be written as TypeScript. The examples below are shown as both TypeScript-only, and YAML + TypeScript.
 
 ## Example 1: A simple load test
 
 Run a simple load test using a plain Playwright script (recorded with `playwright codegen` - no Artillery-specific changes required):
 
+
 ```sh
+# Run the TypeScript example:
+npx artillery run browser-load-test.ts
+```
+
+```sh
+# The same example configured with a separate YAML config file:
 npx artillery run browser-load-test.yml
 ```
 
@@ -25,6 +35,12 @@ For every row in the CSV file, we'll load the URL from the first column, and che
 The test will load each page specified in the CSV file, and check that it contains the text
 
 ```sh
+# Run the TypeScript example:
+npx artillery run browser-smoke-test.ts
+```
+
+```sh
+# The same example configured with a separate YAML config file:
 npx artillery run browser-smoke-test.yml
 ```
 
@@ -92,6 +108,10 @@ browser.page_domcontentloaded.dominteractive.https://artillery.io/pro/:
   p99: ...................................................... 1380.5
 ```
 
-## Scale out
+## Scaling browser tests
+
+Running headless browsers in parallel will quickly exhaust CPU and memory of a single machine.
+
+Artillery has built-in support for cloud-native distributed load testing on AWS Fargate or Azure Container Instances.
 
-Want to run 1,000 browsers at the same time? 10,000? more? Run your load tests on AWS Fargate with built-in support in Artillery. See our guide for [Load testing on AWS Fargate](https://www.artillery.io/docs/load-testing-at-scale/aws-fargate) for more information.
+See our guide for [Distributed load testing](https://www.artillery.io/docs/load-testing-at-scale) for more information.

examples/browser-load-testing-playwright/browser-load-test.ts

@@ -0,0 +1,22 @@
+import { checkOutArtilleryCoreConceptsFlow } from './flows.js';
+
+export const config = {
+  target: 'https://www.artillery.io',
+  phases: [
+    {
+      arrivalRate: 1,
+      duration: 10
+    }
+  ],
+  engines: {
+    playwright: {}
+  }
+};
+
+export const scenarios = [
+  {
+    engine: 'playwright',
+    name: 'check_out_core_concepts_scenario',
+    testFunction: checkOutArtilleryCoreConceptsFlow
+  }
+];

examples/browser-load-testing-playwright/browser-smoke-test.ts

@@ -0,0 +1,27 @@
+import { checkPage } from './flows';
+export const config = {
+  target: 'https://www.artillery.io',
+  phases: [
+    {
+      arrivalCount: 1,
+      duration: 1
+    }
+  ],
+  payload: {
+    path: './pages.csv',
+    fields: ['url', 'title'],
+    loadAll: true,
+    name: 'pageChecks'
+  },
+  engines: {
+    playwright: {}
+  }
+};
+
+export const scenarios = [
+  {
+    name: 'smoke_test_page',
+    engine: 'playwright',
+    testFunction: checkPage
+  }
+];

packages/artillery-engine-playwright/index.js

@@ -351,7 +351,8 @@ class PlaywrightEngine {
 
         const fn =
           self.processor[spec.testFunction] ||
-          self.processor[spec.flowFunction];
+          self.processor[spec.flowFunction] ||
+          spec.testFunction;
 
         if (!fn) {
           console.error('Playwright test function not found:', fn);

packages/artillery/lib/artillery-global.js

@@ -17,10 +17,6 @@ async function createGlobalObject(opts) {
   global.artillery._workerThreadSend =
     global.artillery._workerThreadSend || null;
 
-  // TODO: Refactor these special fields away
-  global.artillery.__util = global.artillery.__util || {};
-  global.artillery.__util.parseScript = parseScript;
-  global.artillery.__util.readScript = readScript;
   global.artillery.__createReporter = require('./console-reporter');
 
   global.artillery._exitCode = 0;

packages/artillery/lib/cmds/quick.js

@@ -70,8 +70,8 @@ class QuickCommand extends Command {
       script.scenarios[0].engine = 'ws';
     }
 
-    const tmpf = tmp.fileSync();
-    fs.writeFileSync(tmpf.name, JSON.stringify(script, null, 2), { flag: 'w' });
+    const tmpf = `${tmp.fileSync().name}.yml`;
+    fs.writeFileSync(tmpf, JSON.stringify(script, null, 2), { flag: 'w' });
 
     const runArgs = [];
     if (flags.output) {
@@ -82,7 +82,7 @@ class QuickCommand extends Command {
       runArgs.push('--quiet');
     }
 
-    runArgs.push(`${tmpf.name}`);
+    runArgs.push(tmpf);
 
     RunCommand.run(runArgs);
   }

packages/artillery/lib/cmds/run.js

@@ -448,58 +448,6 @@ RunCommand.runCommandImplementation = async function (flags, argv, args) {
   }
 };
 
-function replaceProcessorIfTypescript(script, scriptPath) {
-  const relativeProcessorPath = script.config.processor;
-  const userExternalPackages = script.config.bundling?.external || [];
-
-  if (!relativeProcessorPath) {
-    return script;
-  }
-  const extensionType = path.extname(relativeProcessorPath);
-
-  if (extensionType != '.ts') {
-    return script;
-  }
-
-  const actualProcessorPath = path.resolve(
-    path.dirname(scriptPath),
-    relativeProcessorPath
-  );
-  const processorFileName = path.basename(actualProcessorPath, extensionType);
-
-  const processorDir = path.dirname(actualProcessorPath);
-  const newProcessorPath = path.join(
-    processorDir,
-    `dist/${processorFileName}.js`
-  );
-
-  //TODO: move require to top of file when Lambda bundle size issue is solved
-  //must be conditionally required for now as this package is removed in Lambda for now to avoid bigger package sizes
-  const esbuild = require('esbuild-wasm');
-
-  try {
-    esbuild.buildSync({
-      entryPoints: [actualProcessorPath],
-      outfile: newProcessorPath,
-      bundle: true,
-      platform: 'node',
-      format: 'cjs',
-      sourcemap: 'inline',
-      external: ['@playwright/test', ...userExternalPackages]
-    });
-  } catch (error) {
-    throw new Error(`Failed to compile Typescript processor\n${error.message}`);
-  }
-
-  global.artillery.hasTypescriptProcessor = newProcessorPath;
-  console.log(
-    `Bundled Typescript file into JS. New processor path: ${newProcessorPath}`
-  );
-
-  script.config.processor = newProcessorPath;
-  return script;
-}
-
 async function sendTelemetry(script, flags, extraProps) {
   if (process.env.WORKER_ID) {
     debug('Telemetry: Running in cloud worker, skipping test run event');

packages/artillery/lib/platform/aws-ecs/legacy/bom.js

@@ -15,6 +15,10 @@ const BUILTIN_ENGINES = require('./plugins').getOfficialEngines();
 const Table = require('cli-table3');
 
 const { resolveConfigTemplates } = require('../../../../util');
+
+const prepareTestExecutionPlan = require('../../../../lib/util/prepare-test-execution-plan');
+const { readScript, parseScript } = require('../../../../util');
+
 // NOTE: Code below presumes that all paths are absolute
 
 //Tests in Fargate run on ubuntu, which uses posix paths
@@ -28,8 +32,22 @@ function createBOM(absoluteScriptPath, extraFiles, opts, callback) {
   A.waterfall(
     [
       A.constant(absoluteScriptPath),
-      global.artillery.__util.readScript,
-      global.artillery.__util.parseScript,
+      async function (scriptPath) {
+        let scriptData;
+        if (scriptPath.toLowerCase().endsWith('.ts')) {
+          scriptData = await prepareTestExecutionPlan(
+            [scriptPath],
+            opts.flags,
+            []
+          );
+          scriptData.config.processor = scriptPath;
+        } else {
+          const data = await readScript(scriptPath);
+          scriptData = await parseScript(data);
+        }
+
+        return scriptData;
+      },
       (scriptData, next) => {
         return next(null, {
           opts: {

packages/artillery/lib/platform/local/artillery-worker-local.js

@@ -104,9 +104,23 @@ class ArtilleryWorker {
     this.state = STATES.preparing;
 
     const { script, payload, options } = opts;
+    let scriptForWorker = script;
+
+    if (script.__transpiledTypeScriptPath && script.__originalScriptPath) {
+      scriptForWorker = {
+        __transpiledTypeScriptPath: script.__transpiledTypeScriptPath,
+        __originalScriptPath: script.__originalScriptPath
+      };
+    }
+
     this.worker.postMessage({
       command: 'prepare',
-      opts: { script, payload, options, testRunId: global.artillery.testRunId }
+      opts: {
+        script: scriptForWorker,
+        payload,
+        options,
+        testRunId: global.artillery.testRunId
+      }
     });
 
     await awaitOnEE(this.workerEvents, 'readyWaiting', 50);

packages/artillery/lib/platform/local/index.js

@@ -52,6 +52,7 @@ class PlatformLocal {
     }
 
     for (const [workerId, w] of Object.entries(this.workers)) {
+      this.opts.cliArgs = this.platformOpts.cliArgs;
       await this.prepareWorker(workerId, {
         script: w.script,
         payload: this.payload,

packages/artillery/lib/platform/local/worker.js

@@ -31,6 +31,8 @@ const EventEmitter = require('eventemitter3');
 const p = require('util').promisify;
 const { loadProcessor } = core.runner.runnerFuncs;
 
+const prepareTestExecutionPlan = require('../../util/prepare-test-execution-plan');
+
 process.env.LOCAL_WORKER_ID = threadId;
 
 parentPort.on('message', onMessage);
@@ -105,7 +107,22 @@ async function prepare(opts) {
     send({ event: 'log', args });
   });
 
-  const { script: _script, payload, options } = opts;
+  let _script;
+  if (
+    opts.script.__transpiledTypeScriptPath &&
+    opts.script.__originalScriptPath
+  ) {
+    // Load and process pre-compiled TypeScript file
+    _script = await prepareTestExecutionPlan(
+      [opts.script.__originalScriptPath],
+      opts.options.cliArgs,
+      []
+    );
+  } else {
+    _script = opts.script;
+  }
+
+  const { payload, options } = opts;
   const script = await loadProcessor(_script, options);
 
   global.artillery.testRunId = opts.testRunId;