Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(reporter): report step params to the reporters (#42357)

Pavel Feldman0530a54680d2
diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md
index 6342b40901..f60e63a5a8 100644
--- a/docs/src/test-api/class-test.md
+++ b/docs/src/test-api/class-test.md
@@ -1844,6 +1844,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
+### option: Test.step.params
+* since: v1.63
+- `params` <[Object]<[string], [any]>>
+
+Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer.
+
## async method: Test.step.skip
* since: v1.50
- returns: <[void]>
@@ -1891,6 +1897,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
+### option: Test.step.skip.params
+* since: v1.63
+- `params` <[Object]<[string], [any]>>
+
+Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer.
+
### option: Test.step.skip.timeout
* since: v1.50
- `timeout` <[float]>
diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md
index 0c18e6f768..f437bab4d7 100644
--- a/docs/src/test-reporter-api/class-teststep.md
+++ b/docs/src/test-reporter-api/class-teststep.md
@@ -41,6 +41,27 @@ Error thrown during the step execution, if any.
Parent step, if any.
+## property: TestStep.params
+* since: v1.63
+- type: ?<[Object]<[string], [any]>>
+
+Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the target
+`locator` and the call arguments such as `url`, while [`method: Test.step`] steps contain the parameters passed by the
+test author.
+
+```js
+// { locator: 'getByRole(\'button\')' }
+await page.getByRole('button').click();
+
+// { url: 'https://example.com' }
+await page.goto('https://example.com');
+
+// { orderId: 42 }
+await test.step('checkout', async () => {
+ // ...
+}, { params: { orderId: 42 } });
+```
+
## property: TestStep.startTime
* since: v1.10
- type: <[Date]>
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 3943730099..04ca1ef854 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -127,6 +127,7 @@ export type StepBeginPayload = {
parentStepId: string | undefined;
title: string;
category: string;
+ params?: Record<string, any>;
wallTime: number; // milliseconds since unix epoch
location?: { file: string, line: number, column: number };
};
diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts
index 08adee0f6b..81eb262f6d 100644
--- a/packages/playwright/src/common/testType.ts
+++ b/packages/playwright/src/common/testType.ts
@@ -274,12 +274,12 @@ export class TestTypeImpl {
suite._use.push({ fixtures, location });
}
- async _step<T>(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
+ async _step<T>(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number, params?: Record<string, any> } = {}): Promise<T> {
const testInfo = currentTestInfo();
if (!testInfo)
throw new Error(`test.step() can only be called from a test`);
await testInfo._onUserStepBegin?.(title);
- const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box });
+ const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box, params: options.params });
return await currentZone().with('stepZone', step).run(async () => {
try {
let result: Awaited<ReturnType<typeof raceAgainstDeadline<T>>> | undefined = undefined;
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 4933dbc88f..6e61a76a85 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -112,11 +112,12 @@ const utilityFixtures: Fixtures<UtilityTestFixtures, UtilityWorkerFixtures> = {
}
// In the general case, create a step for each api call and connect them through the stepId.
+ const params = renderParams(channel.params);
const step = testInfo._addStep({
location: data.frames[0],
category: 'pw:api',
- title: renderTitle(channel.type, channel.method, channel.params, data.title),
- params: channel.params,
+ title: renderTitle(channel.type, channel.method, channel.params, data.title, params?.locator),
+ params,
group: getActionGroup({ type: channel.type, method: channel.method }),
}, tracingGroupSteps[tracingGroupSteps.length - 1]);
data.stepId = step.stepId;
@@ -919,12 +920,26 @@ function createTestOverlay(parts: string[], position: string, fontSize: number)
</div>`;
}
-function renderTitle(type: string, method: string, params: Record<string, string> | undefined, title?: string) {
+function renderTitle(type: string, method: string, params: Record<string, string> | undefined, title: string | undefined, locator: string | undefined) {
const prefix = renderTitleForCall({ title, type, method, params });
- let selector;
- if (params?.['selector'] && typeof params.selector === 'string')
- selector = asLocatorDescription('javascript', params.selector);
- return prefix + (selector ? ` ${selector}` : '');
+ return prefix + (locator ? ` ${locator}` : '');
+}
+
+const kIncludedParams = new Set(['url', 'selector']);
+
+function renderParams(params: Record<string, any> | undefined): Record<string, any> | undefined {
+ if (!params)
+ return undefined;
+ const result: Record<string, any> = {};
+ for (const [name, value] of Object.entries(params)) {
+ if (!kIncludedParams.has(name))
+ continue;
+ if (name === 'selector' && typeof value === 'string')
+ result.locator = asLocatorDescription('javascript', value);
+ else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')
+ result[name] = value;
+ }
+ return Object.keys(result).length ? result : undefined;
}
function tracing() {
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index b8a15723b8..441c6d24ac 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -109,6 +109,7 @@ export type JsonTestStepStart = {
parentStepId?: string;
title: string;
category: string,
+ params?: Record<string, any>;
startTime: number;
location?: reporterTypes.Location;
};
@@ -724,6 +725,7 @@ export class TeleTestCase implements reporterTypes.TestCase {
class TeleTestStep implements reporterTypes.TestStep {
title: string;
category: string;
+ params: Record<string, any> | undefined;
location: reporterTypes.Location | undefined;
parent: reporterTypes.TestStep | undefined;
duration: number = -1;
@@ -738,6 +740,7 @@ class TeleTestStep implements reporterTypes.TestStep {
constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) {
this.title = payload.title;
this.category = payload.category;
+ this.params = payload.params;
this.location = location;
this.parent = parentStep;
this._startTime = payload.startTime;
diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts
index 751e960ce7..c3879cd76a 100644
--- a/packages/playwright/src/matchers/expect.ts
+++ b/packages/playwright/src/matchers/expect.ts
@@ -336,12 +336,15 @@ function callMatcherAsStep(matcherName: string, info: ExpectMetaInfo, actual: un
// This looks like it is unnecessary, but it isn't - we need to filter
// out all the frames that belong to the test runner from caught runtime errors.
const stackFrames = expectConfig().filteredStackTrace(captureRawStack());
+ const params: Record<string, any> = { ...suffixes.params };
+ if (args[0])
+ params.expected = args[0];
const stepData = {
category: 'expect' as const,
title: longTitle,
shortTitle,
location: stackFrames[0],
- params: args[0] ? { expected: args[0] } : undefined,
+ params: Object.keys(params).length ? params : undefined,
};
const step = testInfo?._addStep(stepData);
diff --git a/packages/playwright/src/matchers/matchers.ts b/packages/playwright/src/matchers/matchers.ts
index 2cbe2705d2..e677241172 100644
--- a/packages/playwright/src/matchers/matchers.ts
+++ b/packages/playwright/src/matchers/matchers.ts
@@ -529,14 +529,15 @@ export async function toPass(
return { pass: !this.isNot, message: () => '' };
}
-export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string } {
+export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string, params?: Record<string, any> } {
if (matcherName === 'toHaveScreenshot') {
const title = toHaveScreenshotStepTitle(...args);
return { short: title ? `(${title})` : '' };
}
if (receiver && typeof receiver === 'object' && (receiver as any)._apiName === 'Locator') {
try {
- return { long: ' ' + asLocatorDescription('javascript', (receiver as LocatorEx)._selector) };
+ const locator = asLocatorDescription('javascript', (receiver as LocatorEx)._selector);
+ return { long: ' ' + locator, params: { locator } };
} catch {
}
}
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index 808239bee5..424a8c203b 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -317,6 +317,7 @@ export class TeleReporterEmitter implements ReporterV2 {
parentStepId: (step.parent as any)?.[this._idSymbol],
title: step.title,
category: step.category,
+ params: step.params,
startTime: +step.startTime,
location: this._relativeLocation(step.location),
};
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 0999900ac1..ba997fbffa 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -405,6 +405,7 @@ class JobDispatcher {
},
parent: parentStep,
category: params.category,
+ params: params.params,
startTime: new Date(params.wallTime),
duration: -1,
steps: [],
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index d7d107cd84..0a043b2d61 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -377,6 +377,7 @@ export class TestInfoImpl implements TestInfo {
parentStepId: parentStep ? parentStep.stepId : undefined,
title: step.title,
category: step.category,
+ params: toReportedParams(step.params),
wallTime: Date.now(),
location: step.location,
};
@@ -729,3 +730,13 @@ export class StepSkipError extends Error {
}
const stepSymbol = Symbol('step');
+
+function toReportedParams(params: Record<string, any> | undefined): Record<string, any> | undefined {
+ if (!params)
+ return undefined;
+ try {
+ return JSON.parse(JSON.stringify(params));
+ } catch {
+ return undefined;
+ }
+}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 19e5f1a8e9..fd7b78e2c2 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -6727,7 +6727,7 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
* @param body Step body.
* @param options
*/
- <T>(title: string, body: (step: TestStepInfo) => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
+ <T>(title: string, body: (step: TestStepInfo) => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise<T>;
/**
* Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and
* planned for a near-term fix. Playwright will not run the step. See also
@@ -6755,7 +6755,7 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
* @param body Step body.
* @param options
*/
- skip(title: string, body: (step: TestStepInfo) => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
+ skip(title: string, body: (step: TestStepInfo) => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise<void>;
}
/**
* `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions).
diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts
index 550b27c860..a9634b136a 100644
--- a/packages/playwright/types/testReporter.d.ts
+++ b/packages/playwright/types/testReporter.d.ts
@@ -875,6 +875,28 @@ export interface TestStep {
*/
location?: Location;
+ /**
+ * Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the
+ * target `locator` and the call arguments such as `url`, while
+ * [test.step(title, body[, options])](https://playwright.dev/docs/api/class-test#test-step) steps contain the
+ * parameters passed by the test author.
+ *
+ * ```js
+ * // { locator: 'getByRole(\'button\')' }
+ * await page.getByRole('button').click();
+ *
+ * // { url: 'https://example.com' }
+ * await page.goto('https://example.com');
+ *
+ * // { orderId: 42 }
+ * await test.step('checkout', async () => {
+ * // ...
+ * }, { params: { orderId: 42 } });
+ * ```
+ *
+ */
+ params?: { [key: string]: any; };
+
/**
* Parent step, if any.
*/
diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts
index b7847e7834..4ed1a3874a 100644
--- a/tests/playwright-test/playwright.trace.spec.ts
+++ b/tests/playwright-test/playwright.trace.spec.ts
@@ -1540,3 +1540,22 @@ test('should record test annotations in trace', async ({ runInlineTest }, testIn
{ type: 'note3' },
]);
});
+
+test('should record step params in trace', async ({ runInlineTest }, testInfo) => {
+ const result = await runInlineTest({
+ 'a.spec.ts': `
+ import { test, expect } from '@playwright/test';
+ test('pass', async ({}) => {
+ expect(1).toBe(1);
+ await test.step('my step', async () => {}, { params: { foo: 'bar' } });
+ });
+ `,
+ }, { trace: 'on' });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.passed).toBe(1);
+ const trace = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip'));
+ const actionByTitle = (title: string) => trace.model.actions.find(a => a.title === title)!;
+ expect(actionByTitle('my step').params).toEqual({ foo: 'bar' });
+ expect(actionByTitle('Expect "toBe"').params).toEqual({ expected: '1' });
+});
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index 0f3590b0e0..3a6912791a 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -1256,6 +1256,40 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show
await expect(page.getByText('Expect "toBe"')).toBeVisible();
});
+test('preserve step params', async ({ runInlineTest, mergeReports }) => {
+ const reportDir = test.info().outputPath('blob-report');
+ const files = {
+ 'params-reporter.js': `
+ class ParamsReporter {
+ onStepEnd(test, result, step) {
+ if (step.category === 'test.step' || step.title.startsWith('Navigate'))
+ console.log('%%' + step.title + ' | ' + JSON.stringify(step.params));
+ }
+ }
+ module.exports = ParamsReporter;
+ `,
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: [['blob']]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('test 1', async ({ page }) => {
+ await page.goto('about:blank');
+ await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } });
+ });
+ `,
+ };
+ await runInlineTest(files);
+ const { exitCode, outputLines } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', './params-reporter.js'] });
+ expect(exitCode).toBe(0);
+ expect(outputLines).toEqual([
+ `Navigate to "about:blank" | {"url":"about:blank"}`,
+ `my step | {"foo":"bar","count":7}`,
+ ]);
+});
+
test('support fileName option', async ({ runInlineTest, mergeReports }) => {
const files = (fileSuffix: string) => ({
'playwright.config.ts': `
diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts
index 0a994c6d85..15a8b68116 100644
--- a/tests/playwright-test/test-step.spec.ts
+++ b/tests/playwright-test/test-step.spec.ts
@@ -1861,3 +1861,39 @@ fixture | Fixture "context"
pw:api | Close context
`);
});
+
+test('should report step params', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'reporter.ts': `
+ import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter';
+ export default class MyReporter implements Reporter {
+ onStepEnd(test: TestCase, result: TestResult, step: TestStep) {
+ if (step.location?.file.endsWith('a.test.ts'))
+ console.log('%%' + step.category + ' | ' + step.title + ' | ' + JSON.stringify(step.params));
+ }
+ }
+ `,
+ 'playwright.config.ts': `
+ module.exports = { reporter: './reporter' };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('pass', async ({ page }) => {
+ await page.goto('about:blank');
+ await page.setContent('<button>Click me</button>');
+ await page.getByRole('button').click();
+ await expect(page.getByRole('button')).toBeVisible();
+ await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } });
+ });
+ `
+ }, { reporter: '' });
+
+ expect(result.exitCode).toBe(0);
+ expect(result.outputLines).toEqual([
+ `pw:api | Navigate to "about:blank" | {"url":"about:blank"}`,
+ `pw:api | Set content | undefined`,
+ `pw:api | Click getByRole('button') | {"locator":"getByRole('button')"}`,
+ `expect | Expect "toBeVisible" getByRole('button') | {"locator":"getByRole('button')"}`,
+ `test.step | my step | {"foo":"bar","count":7}`,
+ ]);
+});
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index 92ccd06110..f8d7829f7e 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -197,8 +197,8 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
afterAll(title: string, inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<any> | any): void;
use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
step: {
- <T>(title: string, body: (step: TestStepInfo) => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
- skip(title: string, body: (step: TestStepInfo) => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
+ <T>(title: string, body: (step: TestStepInfo) => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise<T>;
+ skip(title: string, body: (step: TestStepInfo) => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise<void>;
}
expect: Expect<{}>;
extend<T extends {}, W extends {} = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;