Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(reporter): report step params in the chrome://tracing report (#42358)
Pavel Feldmand4e1023f6c03
diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.mdindex f437bab4d7..8ff66f4be5 100644--- a/docs/src/test-reporter-api/class-teststep.md+++ b/docs/src/test-reporter-api/class-teststep.md@@ -56,12 +56,19 @@ await page.getByRole('button').click();// { url: 'https://example.com' }await page.goto('https://example.com');+// { locator: 'getByLabel(\'Password\')', value: 'secret' }+await page.getByLabel('Password').fill('secret');+// { orderId: 42 }await test.step('checkout', async () => {// ...}, { params: { orderId: 42 } });```+To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string values+are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are never+reported.+## property: TestStep.startTime* since: v1.10- type: <[Date]>diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.mdindex 0e049b8c6f..c80500f3b0 100644--- a/docs/src/test-reporters-js.md+++ b/docs/src/test-reporters-js.md@@ -433,19 +433,20 @@ JUnit report supports following configuration options and environment variables:### Chrome tracing reporter-Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, tags, annotations, errors, stdio and paths to the attachment files.+Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, [`property: TestStep.params`], tags, annotations, errors, stdio and paths to the attachment files.```bashnpx playwright test --reporter=chrome-trace```-By default the report is written to `test-results/chrome-trace.json`.+By default the report is written to `test-results/chrome-trace.json`. When the output file name ends with `.gz`, the+report is gzipped on the fly, which both viewers accept and which is worth it for large test runs.```js title="playwright.config.ts"import { defineConfig } from '@playwright/test';export default defineConfig({- reporter: [['chrome-trace', { outputFile: 'chrome-trace.json' }]],+ reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]],});```diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 6e61a76a85..eda97b3fa4 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -112,7 +112,7 @@ const utilityFixtures: Fixtures<UtilityTestFixtures, UtilityWorkerFixtures> = {}// In the general case, create a step for each api call and connect them through the stepId.- const params = renderParams(channel.params);+ const params = renderParams(channel.type, channel.method, channel.params);const step = testInfo._addStep({location: data.frames[0],category: 'pw:api',@@ -925,23 +925,120 @@ function renderTitle(type: string, method: string, params: Record<string, stringreturn prefix + (locator ? ` ${locator}` : '');}-const kIncludedParams = new Set(['url', 'selector']);+const kMaxParamLength = 200;++// Curated per-call parameters, keyed the same way as the protocol metainfo. Only the+// arguments that say what the call actually did are reported, and only when they are+// bounded in size: page content, evaluated expressions, request bodies and the like are+// never reported, and neither are options that repeat their default on every call.+function renderCallParams(type: string, method: string, params: Record<string, any>): Record<string, any> | undefined {+ switch (`${type}.${method}`) {+ case 'APIRequestContext.fetch':+ return { url: params.url, method: params.method };+ case 'Frame.goto':+ case 'Frame.addScriptTag':+ case 'Frame.addStyleTag':+ return { url: params.url };++ case 'Frame.click':+ case 'Frame.dblclick':+ case 'ElementHandle.click':+ case 'ElementHandle.dblclick':+ return { button: params.button, clickCount: params.clickCount, modifiers: params.modifiers, position: params.position };+ case 'Frame.hover':+ case 'Frame.tap':+ case 'ElementHandle.hover':+ case 'ElementHandle.tap':+ return { modifiers: params.modifiers, position: params.position };+ case 'Frame.check':+ case 'Frame.uncheck':+ case 'ElementHandle.check':+ case 'ElementHandle.uncheck':+ return { position: params.position };+ case 'Frame.dragAndDrop':+ return { source: renderLocator(params.source), target: renderLocator(params.target) };+ case 'Frame.fill':+ case 'ElementHandle.fill':+ return { value: params.value };+ case 'Frame.press':+ case 'ElementHandle.press':+ case 'Page.keyboardDown':+ case 'Page.keyboardUp':+ case 'Page.keyboardPress':+ return { key: params.key };+ case 'Frame.type':+ case 'ElementHandle.type':+ case 'Page.keyboardType':+ case 'Page.keyboardInsertText':+ return { text: params.text };+ case 'Frame.dispatchEvent':+ case 'ElementHandle.dispatchEvent':+ return { type: params.type };+ case 'Frame.selectOption':+ case 'ElementHandle.selectOption':+ return { options: params.options };+ case 'Frame.setInputFiles':+ case 'ElementHandle.setInputFiles':+ return { files: params.localPaths };++ case 'Page.mouseClick':+ return { x: params.x, y: params.y, button: params.button, clickCount: params.clickCount };+ case 'Page.mouseMove':+ case 'Page.touchscreenTap':+ return { x: params.x, y: params.y };+ case 'Page.mouseDown':+ case 'Page.mouseUp':+ return { button: params.button, clickCount: params.clickCount };+ case 'Page.mouseWheel':+ return { deltaX: params.deltaX, deltaY: params.deltaY };++ case 'Frame.waitForSelector':+ case 'ElementHandle.waitForSelector':+ case 'ElementHandle.waitForElementState':+ return { state: params.state };+ case 'Frame.waitForTimeout':+ return { timeout: params.waitTimeout };++ case 'Page.emulateMedia':+ return { media: params.media, colorScheme: params.colorScheme, reducedMotion: params.reducedMotion, forcedColors: params.forcedColors, contrast: params.contrast };+ case 'Page.setViewportSize':+ return params.viewportSize;+ case 'Page.screenshot':+ case 'ElementHandle.screenshot':+ return { type: params.type, fullPage: params.fullPage };+ case 'BrowserContext.setOffline':+ return { offline: params.offline };+ case 'Dialog.accept':+ return { promptText: params.promptText };+ case 'Tracing.tracingGroup':+ return { name: params.name };+ }+}-function renderParams(params: Record<string, any> | undefined): Record<string, any> | undefined {+function renderParams(type: string, method: string, params: Record<string, any> | undefined): Record<string, any> | undefined {if (!params)return undefined;const result: Record<string, any> = {};- for (const [name, value] of Object.entries(params)) {- if (!kIncludedParams.has(name))- continue;- if (name === 'selector' && typeof value === 'string')- result.locator = asLocatorDescription('javascript', value);- else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')- result[name] = value;+ const locator = renderLocator(params.selector);+ if (locator !== undefined)+ result.locator = locator;+ for (const [name, value] of Object.entries(renderCallParams(type, method, params) ?? {})) {+ if (value !== undefined)+ result[name] = typeof value === 'string' ? truncateParam(value) : value;}return Object.keys(result).length ? result : undefined;}+function renderLocator(selector: any): string | undefined {+ if (typeof selector !== 'string')+ return undefined;+ return truncateParam(asLocatorDescription('javascript', selector));+}++function truncateParam(value: string): string {+ return value.length > kMaxParamLength ? value.substring(0, kMaxParamLength) + '\u2026' : value;+}+function tracing() {return (test.info() as TestInfoImpl)._tracing;}diff --git a/packages/playwright/src/reporters/chromeTrace.ts b/packages/playwright/src/reporters/chromeTrace.tsindex de268ea5aa..b3def8c83f 100644--- a/packages/playwright/src/reporters/chromeTrace.ts+++ b/packages/playwright/src/reporters/chromeTrace.ts@@ -16,6 +16,7 @@import fs from 'fs';import path from 'path';+import zlib from 'zlib';import { toPosixPath } from '@utils/fileUtils';import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle';@@ -24,6 +25,7 @@ import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOptionimport { stripAnsiEscapes } from '../util';import type { ReporterV2 } from './reporterV2';+import type { Writable } from 'stream';import type { ChromeTraceReporterOptions } from '../../types/test';import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';@@ -209,6 +211,8 @@ class ChromeTraceReporter implements ReporterV2 {private _stepArgs(step: TestStep): Record<string, any> | undefined {const args: Record<string, any> = {};+ if (step.params)+ args.params = step.params;if (step.location)args.location = this._formatLocation(step.location);if (step.annotations.length)@@ -243,13 +247,13 @@ class ChromeTraceReporter implements ReporterV2 {for (const event of this._events)timeOrigin = Math.min(timeOrigin, event.ts);- const traceEvents: TraceEvent[] = [+ const metadataEvents: TraceEvent[] = [{ name: 'process_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { name: 'Playwright Test' } },{ name: 'process_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { sort_index: 0 } },];for (const lane of [...this._laneEndTime.keys()].sort((a, b) => a - b)) {- traceEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } });- traceEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } });+ metadataEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } });+ metadataEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } });}// Sort is stable, so parent slices stay ahead of their children on ties.@@ -258,22 +262,61 @@ class ChromeTraceReporter implements ReporterV2 {event.ts = Math.round((event.ts - timeOrigin) * 1000);if (event.dur !== undefined)event.dur = Math.round(event.dur * 1000);- traceEvents.push(event);}- const report = {- traceEvents,- displayTimeUnit: 'ms',- metadata: {- 'playwright-version': getPlaywrightVersion(),- 'start-time': result.startTime.toISOString(),- 'duration': result.duration,- 'status': result.status,- },+ const metadata = {+ 'playwright-version': getPlaywrightVersion(),+ 'start-time': result.startTime.toISOString(),+ 'duration': result.duration,+ 'status': result.status,};+ // Serialize event by event, the whole report does not have to fit into memory twice.await fs.promises.mkdir(path.dirname(this._resolvedOutputFile), { recursive: true });- await fs.promises.writeFile(this._resolvedOutputFile, JSON.stringify(report));+ const writer = new ChunkWriter(this._resolvedOutputFile);+ await writer.write('{"traceEvents":[');+ let separator = '';+ for (const events of [metadataEvents, this._events]) {+ for (const event of events) {+ await writer.write(separator + JSON.stringify(event));+ separator = ',';+ }+ }+ await writer.write(`],"displayTimeUnit":"ms","metadata":${JSON.stringify(metadata)}}`);+ await writer.close();+ }+}++// Writes into a ".gz" file through a gzip stream, into a plain file otherwise.+class ChunkWriter {+ private _stream: Writable;+ private _closed: Promise<void>;+ private _error: Error | undefined;++ constructor(file: string) {+ const fileStream = fs.createWriteStream(file);+ const gzip = file.endsWith('.gz') ? zlib.createGzip() : undefined;+ gzip?.pipe(fileStream);+ this._stream = gzip ?? fileStream;+ // The file is only complete once the destination closes, which is later than+ // the gzip stream ending.+ this._closed = new Promise(resolve => fileStream.on('close', resolve));+ for (const stream of new Set<Writable>([this._stream, fileStream]))+ stream.on('error', error => this._error ??= error);+ }++ async write(chunk: string) {+ if (this._error)+ throw this._error;+ if (!this._stream.write(chunk))+ await new Promise<void>(resolve => this._stream.once('drain', () => resolve()));+ }++ async close() {+ this._stream.end();+ await this._closed;+ if (this._error)+ throw this._error;}}diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex a9634b136a..0b84a32293 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -888,12 +888,18 @@ export interface TestStep {* // { url: 'https://example.com' }* await page.goto('https://example.com');*+ * // { locator: 'getByLabel(\'Password\')', value: 'secret' }+ * await page.getByLabel('Password').fill('secret');+ ** // { orderId: 42 }* await test.step('checkout', async () => {* // ...* }, { params: { orderId: 42 } });* ```*+ * To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string+ * values are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are+ * never reported.*/params?: { [key: string]: any; };diff --git a/tests/playwright-test/reporter-chrome-trace.spec.ts b/tests/playwright-test/reporter-chrome-trace.spec.tsindex 77f3ca83f8..6a9193b636 100644--- a/tests/playwright-test/reporter-chrome-trace.spec.ts+++ b/tests/playwright-test/reporter-chrome-trace.spec.ts@@ -16,6 +16,7 @@import * as fs from 'fs';import * as path from 'path';+import * as zlib from 'zlib';import { test, expect } from './playwright-test-fixtures';type TraceEvent = {@@ -31,7 +32,9 @@ type TraceEvent = {};function readTrace(baseDir: string, fileName: string = 'test-results/chrome-trace.json') {- return JSON.parse(fs.readFileSync(path.join(baseDir, fileName), 'utf8')) as {+ const file = path.join(baseDir, fileName);+ const content = fileName.endsWith('.gz') ? zlib.gunzipSync(fs.readFileSync(file)).toString('utf8') : fs.readFileSync(file, 'utf8');+ return JSON.parse(content) as {traceEvents: TraceEvent[],displayTimeUnit: string,metadata: any,@@ -189,6 +192,29 @@ test('should report attachment files', async ({ runInlineTest }, testInfo) => {expect(one.args.attachments).toEqual(['inline', expect.stringMatching(/^test-results\/a-one\/attachments\/file-.*\.txt$/)]);});+test('should report step params', async ({ runInlineTest }, testInfo) => {+ const result = await runInlineTest({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('one', async ({ page }) => {+ await page.goto('about:blank');+ await page.setContent('<button>Click me</button>');+ await page.getByRole('button').click();+ await expect(page.getByRole('button')).toBeVisible();+ await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } });+ });+ `,+ }, { reporter: 'chrome-trace' });+ expect(result.exitCode).toBe(0);++ const events = slices(readTrace(testInfo.outputPath()).traceEvents);+ expect(findSlice(events, 'Navigate to "about:blank"')!.args.params).toEqual({ url: 'about:blank' });+ expect(findSlice(events, 'Set content')!.args.params).toBe(undefined);+ expect(findSlice(events, `Click getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` });+ expect(findSlice(events, `Expect "toBeVisible" getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` });+ expect(findSlice(events, 'my step')!.args.params).toEqual({ foo: 'bar', count: 7 });+});+test('should respect outputFile option', async ({ runInlineTest }, testInfo) => {const result = await runInlineTest({'playwright.config.ts': `@@ -203,6 +229,23 @@ test('should respect outputFile option', async ({ runInlineTest }, testInfo) =>expect(findSlice(readTrace(testInfo.outputPath(), 'reports/my-trace.json').traceEvents, 'one')).toBeTruthy();});+test('should gzip the report when output file ends with .gz', async ({ runInlineTest }, testInfo) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = { reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]] };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}) => {});+ `,+ });+ expect(result.exitCode).toBe(0);++ const gzipped = fs.readFileSync(testInfo.outputPath('chrome-trace.json.gz'));+ expect(gzipped.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b]));+ expect(findSlice(readTrace(testInfo.outputPath(), 'chrome-trace.json.gz').traceEvents, 'one')).toBeTruthy();+});+test('should respect PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => {const result = await runInlineTest({'a.test.ts': `diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.tsindex 15a8b68116..da821b6f25 100644--- a/tests/playwright-test/test-step.spec.ts+++ b/tests/playwright-test/test-step.spec.ts@@ -1897,3 +1897,82 @@ test('should report step params', async ({ runInlineTest }) => {`test.step | my step | {"foo":"bar","count":7}`,]);});++test('should report input step params', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter';+ export default class MyReporter implements Reporter {+ onStepEnd(test: TestCase, result: TestResult, step: TestStep) {+ if (step.location?.file.endsWith('a.test.ts'))+ console.log('%%' + step.title + ' | ' + JSON.stringify(step.params));+ }+ }+ `,+ 'playwright.config.ts': `+ module.exports = { reporter: './reporter' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('pass', async ({ page }) => {+ await page.setContent('<input id=i><button>Click me</button><select><option value=a>a</option><option value=b>b</option></select>');+ await page.locator('#i').fill('value');+ await page.locator('#i').press('Enter');+ await page.keyboard.type('typed');+ await page.getByRole('button').click({ button: 'right', clickCount: 2, modifiers: ['Shift'], position: { x: 3, y: 4 } });+ await page.mouse.move(10, 20);+ await page.mouse.wheel(0, 100);+ await page.locator('select').selectOption('b');+ await page.dispatchEvent('#i', 'focus');+ await page.locator('#i').waitFor({ state: 'visible' });+ await page.setViewportSize({ width: 800, height: 600 });+ await page.dragAndDrop('#i', 'select');+ await page.evaluate(() => 1);+ });+ `+ }, { reporter: '' });++ expect(result.exitCode).toBe(0);+ expect(result.outputLines).toEqual([+ `Set content | undefined`,+ `Fill "value" locator('#i') | {"locator":"locator('#i')","value":"value"}`,+ `Press "Enter" locator('#i') | {"locator":"locator('#i')","key":"Enter"}`,+ `Type "typed" | {"text":"typed"}`,+ `Click getByRole('button') | {"locator":"getByRole('button')","button":"right","clickCount":2,"modifiers":["Shift"],"position":{"x":3,"y":4}}`,+ `Mouse move | {"x":10,"y":20}`,+ `Mouse wheel | {"deltaX":0,"deltaY":100}`,+ `Select option locator('select') | {"locator":"locator('select')","options":[{"valueOrLabel":"b"}]}`,+ `Dispatch "focus" locator('#i') | {"locator":"locator('#i')","type":"focus"}`,+ `Wait for selector locator('#i') | {"locator":"locator('#i')","state":"visible"}`,+ `Set viewport size | {"width":800,"height":600}`,+ `Drag and drop | {"source":"locator('#i')","target":"locator('select')"}`,+ `Evaluate | undefined`,+ ]);+});++test('should truncate long step params', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'reporter.ts': `+ import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter';+ export default class MyReporter implements Reporter {+ onStepEnd(test: TestCase, result: TestResult, step: TestStep) {+ if (step.params?.value)+ console.log('%%' + step.params.value);+ }+ }+ `,+ 'playwright.config.ts': `+ module.exports = { reporter: './reporter' };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('pass', async ({ page }) => {+ await page.setContent('<input id=i>');+ await page.locator('#i').fill('x'.repeat(1000));+ });+ `+ }, { reporter: '' });++ expect(result.exitCode).toBe(0);+ expect(result.outputLines).toEqual(['x'.repeat(200) + '\u2026']);+});