Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(expect): add TestInfoError.matcherResult with ariaSnapshot (#40410)
Pavel Feldman815d051a181e
diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.mdindex eadcaff0fd..ff99b75c5d 100644--- a/docs/src/test-api/class-testinfoerror.md+++ b/docs/src/test-api/class-testinfoerror.md@@ -22,6 +22,19 @@ Error message. Set when [Error] (or its subclass) has been thrown.Error stack. Set when [Error] (or its subclass) has been thrown.+## property: TestInfoError.matcherResult+* since: v1.60+- type: ?<[Object]>+ - `name` <[string]> Matcher name (e.g. `toBeVisible`).+ - `pass` <[boolean]> Whether the matcher passed.+ - `expected` ?<[string]> Expected value formatted as a string.+ - `actual` ?<[string]> Received value formatted as a string.+ - `log` ?<[Array]<[string]>> Matcher log lines, if any.+ - `timeout` ?<[int]> Matcher timeout in milliseconds, set when the matcher timed out.+ - `ariaSnapshot` ?<[string]> Aria snapshot of the receiver at the time of failure, if available.++Structured information about a matcher failure. Set when the error originated from an `expect(...)` matcher; unset otherwise.+## property: TestInfoError.value* since: v1.10- type: ?<[string]>diff --git a/packages/playwright/src/errorContext.ts b/packages/playwright/src/errorContext.tsindex 32baf69c7e..3291232dd5 100644--- a/packages/playwright/src/errorContext.ts+++ b/packages/playwright/src/errorContext.ts@@ -62,6 +62,15 @@ export function buildErrorContext(options: {stripAnsiEscapes(error.message || ''),'```',);+ const ariaSnapshot = error.matcherResult?.ariaSnapshot;+ if (ariaSnapshot) {+ lines.push(+ '',+ '```yaml',+ ariaSnapshot,+ '```',+ );+ }}}diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex 2801495d49..62be6cb69f 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -728,6 +728,8 @@ class ArtifactsRecorder {return;if (this._testInfo.errors.length === 0)return;+ if (this._testInfo.errors.some(e => e.matcherResult?.ariaSnapshot))+ return;if (this._pageSnapshot)return;const page = context.pages()[0];@@ -774,11 +776,12 @@ class ArtifactsRecorder {await this._takePageSnapshot(context);if (this._testInfo.errors.length > 0) {+ const hasMatcherAriaSnapshot = this._testInfo.errors.some(e => e.matcherResult?.ariaSnapshot);const errorContextContent = buildErrorContext({titlePath: this._testInfo.titlePath,location: { file: this._testInfo.file, line: this._testInfo.line, column: this._testInfo.column },errors: this._testInfo.errors,- pageSnapshot: this._pageSnapshot,+ pageSnapshot: hasMatcherAriaSnapshot ? undefined : this._pageSnapshot,});if (errorContextContent) {const filePath = this._testInfo.outputPath('error-context.md');diff --git a/packages/playwright/src/matchers/matcherHint.ts b/packages/playwright/src/matchers/matcherHint.tsindex 8217718bcd..bf8dd14fd6 100644--- a/packages/playwright/src/matchers/matcherHint.ts+++ b/packages/playwright/src/matchers/matcherHint.ts@@ -31,6 +31,7 @@ export type MatcherResult<E = unknown, A = unknown> = {diff?: string;log?: string[];timeout?: number;+ ariaSnapshot?: string;suggestedRebaseline?: string;attachments?: MatcherAttachment[];softError?: Error | unknown;@@ -132,7 +133,6 @@ type MatcherMessageDetails = {timeout?: number;errorMessage?: string;log?: string[];- ariaSnapshot?: string;};export function formatMatcherMessage(utils: InternalMatcherUtils, details: MatcherMessageDetails) {@@ -170,8 +170,6 @@ export function formatMatcherMessage(utils: InternalMatcherUtils, details: Matchmessage += '\n';}message += callLogText(utils, details.log);- if (details.ariaSnapshot)- message += `\nAria snapshot:\n${utils.DIM_COLOR(details.ariaSnapshot)}\n`;return message;}diff --git a/packages/playwright/src/matchers/toBeTruthy.ts b/packages/playwright/src/matchers/toBeTruthy.tsindex 15ad6c4f49..00a0d3b450 100644--- a/packages/playwright/src/matchers/toBeTruthy.ts+++ b/packages/playwright/src/matchers/toBeTruthy.ts@@ -69,7 +69,6 @@ export async function toBeTruthy(printedReceived,errorMessage,log,- ariaSnapshot: received?.ariaSnapshot,});};@@ -81,5 +80,6 @@ export async function toBeTruthy(expected,log,timeout: timedOut ? timeout : undefined,+ ariaSnapshot: received?.ariaSnapshot,};}diff --git a/packages/playwright/src/matchers/toEqual.ts b/packages/playwright/src/matchers/toEqual.tsindex 0b45dce8ef..73528f2466 100644--- a/packages/playwright/src/matchers/toEqual.ts+++ b/packages/playwright/src/matchers/toEqual.ts@@ -98,7 +98,6 @@ export async function toEqual<T>(printedDiff,errorMessage,log,- ariaSnapshot: received?.ariaSnapshot,});};@@ -112,5 +111,6 @@ export async function toEqual<T>(pass,log,timeout: timedOut ? timeout : undefined,+ ariaSnapshot: received?.ariaSnapshot,};}diff --git a/packages/playwright/src/matchers/toMatchText.ts b/packages/playwright/src/matchers/toMatchText.tsindex 0984943a36..7771592729 100644--- a/packages/playwright/src/matchers/toMatchText.ts+++ b/packages/playwright/src/matchers/toMatchText.ts@@ -96,7 +96,6 @@ export async function toMatchText(printedDiff,log,errorMessage,- ariaSnapshot: received?.ariaSnapshot,});};@@ -108,5 +107,6 @@ export async function toMatchText(actual: receivedValue,log,timeout: timedOut ? timeout : undefined,+ ariaSnapshot: received?.ariaSnapshot,};}diff --git a/packages/playwright/src/worker/util.ts b/packages/playwright/src/worker/util.tsindex f25dd954c4..47b337ff26 100644--- a/packages/playwright/src/worker/util.ts+++ b/packages/playwright/src/worker/util.ts@@ -14,10 +14,38 @@* limitations under the License.*/+import util from 'util';+import { serializeError } from '../util';import type { ipc } from '../common';+import type { MatcherResultProperty } from '../matchers/matcherHint';export function testInfoError(error: Error | any): ipc.TestInfoErrorImpl {- return serializeError(error);+ const result = serializeError(error);+ const matcherResult = (error instanceof Error ? (error as any).matcherResult : undefined) as MatcherResultProperty | undefined;+ if (matcherResult) {+ const serialized: NonNullable<ipc.TestInfoErrorImpl['matcherResult']> = {+ name: matcherResult.name,+ pass: matcherResult.pass,+ };+ if (matcherResult.expected !== undefined)+ serialized.expected = inspectForIpc(matcherResult.expected);+ if (matcherResult.actual !== undefined)+ serialized.actual = inspectForIpc(matcherResult.actual);+ if (matcherResult.log !== undefined)+ serialized.log = matcherResult.log;+ if (matcherResult.timeout !== undefined)+ serialized.timeout = matcherResult.timeout;+ if (matcherResult.ariaSnapshot !== undefined)+ serialized.ariaSnapshot = matcherResult.ariaSnapshot;+ result.matcherResult = serialized;+ }+ return result;+}++function inspectForIpc(value: unknown): string {+ if (typeof value === 'string')+ return value;+ return util.inspect(value, { depth: 10 });}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex 40bfb4f348..3be43868f3 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -10007,6 +10007,47 @@ export interface TestInfoError {*/cause?: TestInfoError;+ /**+ * Structured information about a matcher failure. Set when the error originated from an `expect(...)` matcher; unset+ * otherwise.+ */+ matcherResult?: {+ /**+ * Matcher name (e.g. `toBeVisible`).+ */+ name: string;++ /**+ * Whether the matcher passed.+ */+ pass: boolean;++ /**+ * Expected value formatted as a string.+ */+ expected?: string;++ /**+ * Received value formatted as a string.+ */+ actual?: string;++ /**+ * Matcher log lines, if any.+ */+ log?: Array<string>;++ /**+ * Matcher timeout in milliseconds, set when the matcher timed out.+ */+ timeout?: number;++ /**+ * Aria snapshot of the receiver at the time of failure, if available.+ */+ ariaSnapshot?: string;+ };+/*** Error message. Set when [Error] (or its subclass) has been thrown.*/diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.tsindex d05f3c54c9..c81ed7825e 100644--- a/tests/mcp/test-debug.spec.ts+++ b/tests/mcp/test-debug.spec.ts@@ -67,9 +67,6 @@ Call log:- Expect "toBeVisible" with timeout 1000ms- waiting for getByRole('button', { name: 'Missing' })-Aria snapshot:-- button "Submit"-### Page state- Page URL: about:blank@@ -189,9 +186,6 @@ Call log:- Expect "toBeVisible" with timeout 1000ms- waiting for getByRole('button', { name: 'Missing' })-Aria snapshot:-- button "Submit"-### Page state- Page URL: about:blankdiff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.tsindex e4e6f28cdf..455ac99ee4 100644--- a/tests/page/expect-matcher-result.spec.ts+++ b/tests/page/expect-matcher-result.spec.ts@@ -32,6 +32,7 @@ test('toMatchText-based assertions should have matcher result', async ({ page })pass: false,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toHaveText(expected) failed@@ -56,6 +57,7 @@ Call log`);pass: true,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toHaveText(expected) failed@@ -83,6 +85,7 @@ test('toBeTruthy-based assertions should have matcher result', async ({ page })pass: false,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeVisible() failed@@ -108,6 +111,7 @@ Call log:pass: true,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeVisible() failed@@ -190,6 +194,7 @@ test('toBeChecked({ checked }) should have expected', async ({ page }) => {pass: false,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked() failed@@ -214,6 +219,7 @@ Call log`);pass: true,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeChecked() failed@@ -238,6 +244,7 @@ Call log`);pass: false,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked({ checked: false }) failed@@ -262,6 +269,7 @@ Call log`);pass: true,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeChecked({ checked: false }) failed@@ -286,6 +294,7 @@ Call log`);pass: false,log: expect.any(Array),timeout: 1,+ ariaSnapshot: expect.any(String),});expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked({ indeterminate: true }) faileddiff --git a/tests/page/expect-with-snapshot.spec.ts b/tests/page/expect-with-snapshot.spec.tsindex e16e2e52f7..8c02d2158b 100644--- a/tests/page/expect-with-snapshot.spec.ts+++ b/tests/page/expect-with-snapshot.spec.ts@@ -14,27 +14,24 @@* limitations under the License.*/-import { stripAnsi } from '../config/utils';import { test, expect } from './pageTest';test.describe('containment matchers print full element subtree', () => {test('toHaveText failure includes full element subtree', async ({ page }) => {await page.setContent(`<section id=node><h1>Title</h1><p>Body</p></section>`);const error = await expect(page.locator('#node')).toHaveText('nope', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('heading "Title"');- expect(message).toContain('paragraph');- expect(message).toContain('Body');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('heading "Title"');+ expect(snapshot).toContain('paragraph');+ expect(snapshot).toContain('Body');});test('toContainText failure includes full element subtree', async ({ page }) => {await page.setContent(`<section id=node><h1>Title</h1><p>Body</p></section>`);const error = await expect(page.locator('#node')).toContainText('nope', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('heading "Title"');- expect(message).toContain('Body');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('heading "Title"');+ expect(snapshot).toContain('Body');});});@@ -42,45 +39,40 @@ test.describe('property matchers print only the element line', () => {test('toBeChecked failure prints just the input', async ({ page }) => {await page.setContent(`<label><input id=cb type=checkbox> a checkbox</label>`);const error = await expect(page.locator('#cb')).toBeChecked({ timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('checkbox');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('checkbox');});test('toHaveAttribute failure clips descendant subtree', async ({ page }) => {await page.setContent(`<ul id=lst><li><h2>HeadingMarker</h2><p>BodyMarker</p></li></ul>`);const error = await expect(page.locator('#lst')).toHaveAttribute('data-x', 'yes', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('list');- expect(message).toContain('listitem');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('list');+ expect(snapshot).toContain('listitem');// Property matcher caps depth at 1 — content inside the listitem (depth 2) must not appear.- expect(message).not.toContain('HeadingMarker');- expect(message).not.toContain('BodyMarker');+ expect(snapshot).not.toContain('HeadingMarker');+ expect(snapshot).not.toContain('BodyMarker');});test('toHaveRole failure prints just the element line', async ({ page }) => {await page.setContent(`<button id=btn>Hi<span>nested</span></button>`);const error = await expect(page.locator('#btn')).toHaveRole('link', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('button');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('button');});test('toHaveValue failure prints the input element', async ({ page }) => {await page.setContent(`<input id=inp value="actual">`);const error = await expect(page.locator('#inp')).toHaveValue('expected', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('textbox');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('textbox');});test('toHaveCSS failure prints the element line', async ({ page }) => {await page.setContent(`<button id=btn style="color: red">Press</button>`);const error = await expect(page.locator('#btn')).toHaveCSS('color', 'rgb(0, 0, 0)', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('button');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('button');});});@@ -91,34 +83,30 @@ test.describe('hidden or missing elements print full page snapshot', () => {<main><h1>Page Heading</h1></main>`);const error = await expect(page.locator('#hidden')).toBeVisible({ timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');+ const snapshot = error.matcherResult.ariaSnapshot;// Page-wide context, not the empty hidden element snapshot.- expect(message).toContain('heading "Page Heading"');+ expect(snapshot).toContain('heading "Page Heading"');});test('toBeVisible on missing element prints full page snapshot', async ({ page }) => {await page.setContent(`<header><h1>Hello</h1></header>`);const error = await expect(page.locator('#nope')).toBeVisible({ timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('heading "Hello"');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('heading "Hello"');});test('toHaveText on missing element prints full page snapshot', async ({ page }) => {await page.setContent(`<main><h1>Hello</h1></main>`);const error = await expect(page.locator('#missing')).toHaveText('x', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('heading "Hello"');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('heading "Hello"');});test('toHaveTitle failure prints full page snapshot', async ({ page }) => {await page.setContent(`<title>Right</title><main><h1>Body Heading</h1></main>`);const error = await expect(page).toHaveTitle('Wrong', { timeout: 1000 }).catch(e => e);- const message = stripAnsi(error.message);- expect(message).toContain('Aria snapshot:');- expect(message).toContain('heading "Body Heading"');+ const snapshot = error.matcherResult.ariaSnapshot;+ expect(snapshot).toContain('heading "Body Heading"');});});@@ -126,18 +114,18 @@ test.describe('matchers that should not include an aria snapshot', () => {test('toHaveCount failure has no aria snapshot', async ({ page }) => {await page.setContent(`<ul><li>a</li><li>b</li></ul>`);const error = await expect(page.locator('li')).toHaveCount(5, { timeout: 1000 }).catch(e => e);- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();});test('toHaveText with array has no aria snapshot', async ({ page }) => {await page.setContent(`<ul><li>x</li><li>y</li></ul>`);const error = await expect(page.locator('li')).toHaveText(['a', 'b', 'c'], { timeout: 1000 }).catch(e => e);- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();});- test('toMatchAriaSnapshot failure has no extra aria snapshot section', async ({ page }) => {+ test('toMatchAriaSnapshot failure has no extra aria snapshot', async ({ page }) => {await page.setContent(`<button id=btn>Y</button>`);const error = await expect(page.locator('#btn')).toMatchAriaSnapshot(`- button "X"`, { timeout: 1000 }).catch(e => e);- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();});});