Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(expect): add TestInfoError.matcherResult with ariaSnapshot (#40410)

Pavel Feldman815d051a181e
diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.md
index eadcaff0fd..ff99b75c5d 100644
--- a/docs/src/test-api/class-testinfoerror.md
+++ b/docs/src/test-api/class-testinfoerror.md
@@ -22,6 +22,19 @@ Error message. Set when [Error] (or its subclass) has been thrown.
Error stack. Set when [Error] (or its subclass) has been thrown.
+## property: TestInfoError.matcherResult
+* since: v1.60
+- type: ?<[Object]>
+ - `name` <[string]> Matcher name (e.g. `toBeVisible`).
+ - `pass` <[boolean]> Whether the matcher passed.
+ - `expected` ?<[string]> Expected value formatted as a string.
+ - `actual` ?<[string]> Received value formatted as a string.
+ - `log` ?<[Array]<[string]>> Matcher log lines, if any.
+ - `timeout` ?<[int]> Matcher timeout in milliseconds, set when the matcher timed out.
+ - `ariaSnapshot` ?<[string]> Aria snapshot of the receiver at the time of failure, if available.
+
+Structured information about a matcher failure. Set when the error originated from an `expect(...)` matcher; unset otherwise.
+
## property: TestInfoError.value
* since: v1.10
- type: ?<[string]>
diff --git a/packages/playwright/src/errorContext.ts b/packages/playwright/src/errorContext.ts
index 32baf69c7e..3291232dd5 100644
--- a/packages/playwright/src/errorContext.ts
+++ b/packages/playwright/src/errorContext.ts
@@ -62,6 +62,15 @@ export function buildErrorContext(options: {
stripAnsiEscapes(error.message || ''),
'```',
);
+ const ariaSnapshot = error.matcherResult?.ariaSnapshot;
+ if (ariaSnapshot) {
+ lines.push(
+ '',
+ '```yaml',
+ ariaSnapshot,
+ '```',
+ );
+ }
}
}
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index 2801495d49..62be6cb69f 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -728,6 +728,8 @@ class ArtifactsRecorder {
return;
if (this._testInfo.errors.length === 0)
return;
+ if (this._testInfo.errors.some(e => e.matcherResult?.ariaSnapshot))
+ return;
if (this._pageSnapshot)
return;
const page = context.pages()[0];
@@ -774,11 +776,12 @@ class ArtifactsRecorder {
await this._takePageSnapshot(context);
if (this._testInfo.errors.length > 0) {
+ const hasMatcherAriaSnapshot = this._testInfo.errors.some(e => e.matcherResult?.ariaSnapshot);
const errorContextContent = buildErrorContext({
titlePath: this._testInfo.titlePath,
location: { file: this._testInfo.file, line: this._testInfo.line, column: this._testInfo.column },
errors: this._testInfo.errors,
- pageSnapshot: this._pageSnapshot,
+ pageSnapshot: hasMatcherAriaSnapshot ? undefined : this._pageSnapshot,
});
if (errorContextContent) {
const filePath = this._testInfo.outputPath('error-context.md');
diff --git a/packages/playwright/src/matchers/matcherHint.ts b/packages/playwright/src/matchers/matcherHint.ts
index 8217718bcd..bf8dd14fd6 100644
--- a/packages/playwright/src/matchers/matcherHint.ts
+++ b/packages/playwright/src/matchers/matcherHint.ts
@@ -31,6 +31,7 @@ export type MatcherResult<E = unknown, A = unknown> = {
diff?: string;
log?: string[];
timeout?: number;
+ ariaSnapshot?: string;
suggestedRebaseline?: string;
attachments?: MatcherAttachment[];
softError?: Error | unknown;
@@ -132,7 +133,6 @@ type MatcherMessageDetails = {
timeout?: number;
errorMessage?: string;
log?: string[];
- ariaSnapshot?: string;
};
export function formatMatcherMessage(utils: InternalMatcherUtils, details: MatcherMessageDetails) {
@@ -170,8 +170,6 @@ export function formatMatcherMessage(utils: InternalMatcherUtils, details: Match
message += '\n';
}
message += callLogText(utils, details.log);
- if (details.ariaSnapshot)
- message += `\nAria snapshot:\n${utils.DIM_COLOR(details.ariaSnapshot)}\n`;
return message;
}
diff --git a/packages/playwright/src/matchers/toBeTruthy.ts b/packages/playwright/src/matchers/toBeTruthy.ts
index 15ad6c4f49..00a0d3b450 100644
--- a/packages/playwright/src/matchers/toBeTruthy.ts
+++ b/packages/playwright/src/matchers/toBeTruthy.ts
@@ -69,7 +69,6 @@ export async function toBeTruthy(
printedReceived,
errorMessage,
log,
- ariaSnapshot: received?.ariaSnapshot,
});
};
@@ -81,5 +80,6 @@ export async function toBeTruthy(
expected,
log,
timeout: timedOut ? timeout : undefined,
+ ariaSnapshot: received?.ariaSnapshot,
};
}
diff --git a/packages/playwright/src/matchers/toEqual.ts b/packages/playwright/src/matchers/toEqual.ts
index 0b45dce8ef..73528f2466 100644
--- a/packages/playwright/src/matchers/toEqual.ts
+++ b/packages/playwright/src/matchers/toEqual.ts
@@ -98,7 +98,6 @@ export async function toEqual<T>(
printedDiff,
errorMessage,
log,
- ariaSnapshot: received?.ariaSnapshot,
});
};
@@ -112,5 +111,6 @@ export async function toEqual<T>(
pass,
log,
timeout: timedOut ? timeout : undefined,
+ ariaSnapshot: received?.ariaSnapshot,
};
}
diff --git a/packages/playwright/src/matchers/toMatchText.ts b/packages/playwright/src/matchers/toMatchText.ts
index 0984943a36..7771592729 100644
--- a/packages/playwright/src/matchers/toMatchText.ts
+++ b/packages/playwright/src/matchers/toMatchText.ts
@@ -96,7 +96,6 @@ export async function toMatchText(
printedDiff,
log,
errorMessage,
- ariaSnapshot: received?.ariaSnapshot,
});
};
@@ -108,5 +107,6 @@ export async function toMatchText(
actual: receivedValue,
log,
timeout: timedOut ? timeout : undefined,
+ ariaSnapshot: received?.ariaSnapshot,
};
}
diff --git a/packages/playwright/src/worker/util.ts b/packages/playwright/src/worker/util.ts
index f25dd954c4..47b337ff26 100644
--- a/packages/playwright/src/worker/util.ts
+++ b/packages/playwright/src/worker/util.ts
@@ -14,10 +14,38 @@
* limitations under the License.
*/
+import util from 'util';
+
import { serializeError } from '../util';
import type { ipc } from '../common';
+import type { MatcherResultProperty } from '../matchers/matcherHint';
export function testInfoError(error: Error | any): ipc.TestInfoErrorImpl {
- return serializeError(error);
+ const result = serializeError(error);
+ const matcherResult = (error instanceof Error ? (error as any).matcherResult : undefined) as MatcherResultProperty | undefined;
+ if (matcherResult) {
+ const serialized: NonNullable<ipc.TestInfoErrorImpl['matcherResult']> = {
+ name: matcherResult.name,
+ pass: matcherResult.pass,
+ };
+ if (matcherResult.expected !== undefined)
+ serialized.expected = inspectForIpc(matcherResult.expected);
+ if (matcherResult.actual !== undefined)
+ serialized.actual = inspectForIpc(matcherResult.actual);
+ if (matcherResult.log !== undefined)
+ serialized.log = matcherResult.log;
+ if (matcherResult.timeout !== undefined)
+ serialized.timeout = matcherResult.timeout;
+ if (matcherResult.ariaSnapshot !== undefined)
+ serialized.ariaSnapshot = matcherResult.ariaSnapshot;
+ result.matcherResult = serialized;
+ }
+ return result;
+}
+
+function inspectForIpc(value: unknown): string {
+ if (typeof value === 'string')
+ return value;
+ return util.inspect(value, { depth: 10 });
}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index 40bfb4f348..3be43868f3 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -10007,6 +10007,47 @@ export interface TestInfoError {
*/
cause?: TestInfoError;
+ /**
+ * Structured information about a matcher failure. Set when the error originated from an `expect(...)` matcher; unset
+ * otherwise.
+ */
+ matcherResult?: {
+ /**
+ * Matcher name (e.g. `toBeVisible`).
+ */
+ name: string;
+
+ /**
+ * Whether the matcher passed.
+ */
+ pass: boolean;
+
+ /**
+ * Expected value formatted as a string.
+ */
+ expected?: string;
+
+ /**
+ * Received value formatted as a string.
+ */
+ actual?: string;
+
+ /**
+ * Matcher log lines, if any.
+ */
+ log?: Array<string>;
+
+ /**
+ * Matcher timeout in milliseconds, set when the matcher timed out.
+ */
+ timeout?: number;
+
+ /**
+ * Aria snapshot of the receiver at the time of failure, if available.
+ */
+ ariaSnapshot?: string;
+ };
+
/**
* Error message. Set when [Error] (or its subclass) has been thrown.
*/
diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.ts
index d05f3c54c9..c81ed7825e 100644
--- a/tests/mcp/test-debug.spec.ts
+++ b/tests/mcp/test-debug.spec.ts
@@ -67,9 +67,6 @@ Call log:
- Expect "toBeVisible" with timeout 1000ms
- waiting for getByRole('button', { name: 'Missing' })
-Aria snapshot:
-- button "Submit"
-
### Page state
- Page URL: about:blank
@@ -189,9 +186,6 @@ Call log:
- Expect "toBeVisible" with timeout 1000ms
- waiting for getByRole('button', { name: 'Missing' })
-Aria snapshot:
-- button "Submit"
-
### Page state
- Page URL: about:blank
diff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.ts
index e4e6f28cdf..455ac99ee4 100644
--- a/tests/page/expect-matcher-result.spec.ts
+++ b/tests/page/expect-matcher-result.spec.ts
@@ -32,6 +32,7 @@ test('toMatchText-based assertions should have matcher result', async ({ page })
pass: false,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toHaveText(expected) failed
@@ -56,6 +57,7 @@ Call log`);
pass: true,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toHaveText(expected) failed
@@ -83,6 +85,7 @@ test('toBeTruthy-based assertions should have matcher result', async ({ page })
pass: false,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeVisible() failed
@@ -108,6 +111,7 @@ Call log:
pass: true,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeVisible() failed
@@ -190,6 +194,7 @@ test('toBeChecked({ checked }) should have expected', async ({ page }) => {
pass: false,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked() failed
@@ -214,6 +219,7 @@ Call log`);
pass: true,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeChecked() failed
@@ -238,6 +244,7 @@ Call log`);
pass: false,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked({ checked: false }) failed
@@ -262,6 +269,7 @@ Call log`);
pass: true,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).not.toBeChecked({ checked: false }) failed
@@ -286,6 +294,7 @@ Call log`);
pass: false,
log: expect.any(Array),
timeout: 1,
+ ariaSnapshot: expect.any(String),
});
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(locator).toBeChecked({ indeterminate: true }) failed
diff --git a/tests/page/expect-with-snapshot.spec.ts b/tests/page/expect-with-snapshot.spec.ts
index e16e2e52f7..8c02d2158b 100644
--- a/tests/page/expect-with-snapshot.spec.ts
+++ b/tests/page/expect-with-snapshot.spec.ts
@@ -14,27 +14,24 @@
* limitations under the License.
*/
-import { stripAnsi } from '../config/utils';
import { test, expect } from './pageTest';
test.describe('containment matchers print full element subtree', () => {
test('toHaveText failure includes full element subtree', async ({ page }) => {
await page.setContent(`<section id=node><h1>Title</h1><p>Body</p></section>`);
const error = await expect(page.locator('#node')).toHaveText('nope', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('heading "Title"');
- expect(message).toContain('paragraph');
- expect(message).toContain('Body');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('heading "Title"');
+ expect(snapshot).toContain('paragraph');
+ expect(snapshot).toContain('Body');
});
test('toContainText failure includes full element subtree', async ({ page }) => {
await page.setContent(`<section id=node><h1>Title</h1><p>Body</p></section>`);
const error = await expect(page.locator('#node')).toContainText('nope', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('heading "Title"');
- expect(message).toContain('Body');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('heading "Title"');
+ expect(snapshot).toContain('Body');
});
});
@@ -42,45 +39,40 @@ test.describe('property matchers print only the element line', () => {
test('toBeChecked failure prints just the input', async ({ page }) => {
await page.setContent(`<label><input id=cb type=checkbox> a checkbox</label>`);
const error = await expect(page.locator('#cb')).toBeChecked({ timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('checkbox');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('checkbox');
});
test('toHaveAttribute failure clips descendant subtree', async ({ page }) => {
await page.setContent(`<ul id=lst><li><h2>HeadingMarker</h2><p>BodyMarker</p></li></ul>`);
const error = await expect(page.locator('#lst')).toHaveAttribute('data-x', 'yes', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('list');
- expect(message).toContain('listitem');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('list');
+ expect(snapshot).toContain('listitem');
// Property matcher caps depth at 1 — content inside the listitem (depth 2) must not appear.
- expect(message).not.toContain('HeadingMarker');
- expect(message).not.toContain('BodyMarker');
+ expect(snapshot).not.toContain('HeadingMarker');
+ expect(snapshot).not.toContain('BodyMarker');
});
test('toHaveRole failure prints just the element line', async ({ page }) => {
await page.setContent(`<button id=btn>Hi<span>nested</span></button>`);
const error = await expect(page.locator('#btn')).toHaveRole('link', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('button');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('button');
});
test('toHaveValue failure prints the input element', async ({ page }) => {
await page.setContent(`<input id=inp value="actual">`);
const error = await expect(page.locator('#inp')).toHaveValue('expected', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('textbox');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('textbox');
});
test('toHaveCSS failure prints the element line', async ({ page }) => {
await page.setContent(`<button id=btn style="color: red">Press</button>`);
const error = await expect(page.locator('#btn')).toHaveCSS('color', 'rgb(0, 0, 0)', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('button');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('button');
});
});
@@ -91,34 +83,30 @@ test.describe('hidden or missing elements print full page snapshot', () => {
<main><h1>Page Heading</h1></main>
`);
const error = await expect(page.locator('#hidden')).toBeVisible({ timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
+ const snapshot = error.matcherResult.ariaSnapshot;
// Page-wide context, not the empty hidden element snapshot.
- expect(message).toContain('heading "Page Heading"');
+ expect(snapshot).toContain('heading "Page Heading"');
});
test('toBeVisible on missing element prints full page snapshot', async ({ page }) => {
await page.setContent(`<header><h1>Hello</h1></header>`);
const error = await expect(page.locator('#nope')).toBeVisible({ timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('heading "Hello"');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('heading "Hello"');
});
test('toHaveText on missing element prints full page snapshot', async ({ page }) => {
await page.setContent(`<main><h1>Hello</h1></main>`);
const error = await expect(page.locator('#missing')).toHaveText('x', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('heading "Hello"');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('heading "Hello"');
});
test('toHaveTitle failure prints full page snapshot', async ({ page }) => {
await page.setContent(`<title>Right</title><main><h1>Body Heading</h1></main>`);
const error = await expect(page).toHaveTitle('Wrong', { timeout: 1000 }).catch(e => e);
- const message = stripAnsi(error.message);
- expect(message).toContain('Aria snapshot:');
- expect(message).toContain('heading "Body Heading"');
+ const snapshot = error.matcherResult.ariaSnapshot;
+ expect(snapshot).toContain('heading "Body Heading"');
});
});
@@ -126,18 +114,18 @@ test.describe('matchers that should not include an aria snapshot', () => {
test('toHaveCount failure has no aria snapshot', async ({ page }) => {
await page.setContent(`<ul><li>a</li><li>b</li></ul>`);
const error = await expect(page.locator('li')).toHaveCount(5, { timeout: 1000 }).catch(e => e);
- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');
+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();
});
test('toHaveText with array has no aria snapshot', async ({ page }) => {
await page.setContent(`<ul><li>x</li><li>y</li></ul>`);
const error = await expect(page.locator('li')).toHaveText(['a', 'b', 'c'], { timeout: 1000 }).catch(e => e);
- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');
+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();
});
- test('toMatchAriaSnapshot failure has no extra aria snapshot section', async ({ page }) => {
+ test('toMatchAriaSnapshot failure has no extra aria snapshot', async ({ page }) => {
await page.setContent(`<button id=btn>Y</button>`);
const error = await expect(page.locator('#btn')).toMatchAriaSnapshot(`- button "X"`, { timeout: 1000 }).catch(e => e);
- expect(stripAnsi(error.message)).not.toContain('Aria snapshot:');
+ expect(error.matcherResult.ariaSnapshot).toBeUndefined();
});
});