Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: group attachments in terminal reporters (#36607)
Pavel Feldman2cbe965880ba
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsxindex 7639e3577b..6fa7bc50ec 100644--- a/packages/html-reporter/src/testErrorView.tsx+++ b/packages/html-reporter/src/testErrorView.tsx@@ -47,16 +47,10 @@ export const PromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {};export const TestScreenshotErrorView: React.FC<{- errorPrefix?: string,diff: ImageDiff,- errorSuffix?: string,-}> = ({ errorPrefix, diff, errorSuffix }) => {- const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]);- const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]);+}> = ({ diff }) => {return <div data-testid='test-screenshot-error-view' className='test-error-view'>- <div dangerouslySetInnerHTML={{ __html: prefixHtml || '' }} className='test-error-text' style={{ marginBottom: 20 }}></div><ImageDiffView key='image-diff' diff={diff} hideDetails={true}></ImageDiffView>- <div data-testid='error-suffix' dangerouslySetInnerHTML={{ __html: suffixHtml || '' }} className='test-error-text'></div></div>;};diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsxindex b023c27981..c8f8f04029 100644--- a/packages/html-reporter/src/testResultView.tsx+++ b/packages/html-reporter/src/testResultView.tsx@@ -86,7 +86,7 @@ export const TestResultView: React.FC<{[...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`);const diffs = groupImageDiffs(screenshots, result);- const errors = classifyErrors(result.errors.map(e => e.message), diffs);+ const errors = result.errors.map(e => e.message);return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext };}, [result]);@@ -111,9 +111,11 @@ export const TestResultView: React.FC<{</div>)}{errors.map((error, index) => {- if (error.type === 'screenshot')- return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;- return <CodeSnippet key={'test-result-error-message-' + index} code={error.error!}/>;+ const diff = pickDiffForError(error, diffs);+ return <>+ <CodeSnippet key={'test-result-error-message-' + index} code={error}/>+ {diff && <TestScreenshotErrorView diff={diff}></TestScreenshotErrorView>}+ </>;})}</AutoChip>}{!!result.steps.length && <AutoChip header='Test Steps'>@@ -167,29 +169,11 @@ export const TestResultView: React.FC<{</div>;};-function classifyErrors(testErrors: string[], diffs: ImageDiff[]) {- return testErrors.map(error => {- const firstLine = error.split('\n')[0];- if (firstLine.includes('toHaveScreenshot') || firstLine.includes('toMatchSnapshot')) {- const matchingDiff = diffs.find(diff => {- const attachmentName = diff.actual?.attachment.name;- return attachmentName && error.includes(attachmentName);- });-- if (matchingDiff) {- const lines = error.split('\n');- const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line));- const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0];-- const diffIndex = lines.findIndex(line => / +Diff:/.test(line));- const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n');-- return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix };- }- }-- return { type: 'regular', error };- });+function pickDiffForError(error: string, diffs: ImageDiff[]): ImageDiff | undefined {+ const firstLine = error.split('\n')[0];+ if (!firstLine.includes('toHaveScreenshot') && !firstLine.includes('toMatchSnapshot'))+ return undefined;+ return diffs.find(diff => error.includes(diff.name));}const StepTreeItem: React.FC<{diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.tsindex b2ef3c3323..ac3e5344b3 100644--- a/packages/playwright/src/matchers/toMatchSnapshot.ts+++ b/packages/playwright/src/matchers/toMatchSnapshot.ts@@ -68,6 +68,7 @@ const NonConfigProperties: (keyof ToHaveScreenshotOptions)[] = [class SnapshotHelper {readonly testInfo: TestInfoImpl;+ readonly name: string;readonly attachmentBaseName: string;readonly legacyExpectedPath: string;readonly previousPath: string;@@ -101,6 +102,7 @@ class SnapshotHelper {name = nameFromOptions;}+ this.name = Array.isArray(name) ? name.join(path.sep) : name || '';const resolvedPaths = testInfo._resolveSnapshotPaths(matcherName === 'toHaveScreenshot' ? 'screenshot' : 'snapshot', name, 'updateSnapshotIndex', anonymousSnapshotExtension);this.expectedPath = resolvedPaths.absoluteSnapshotPath;this.attachmentBaseName = resolvedPaths.relativeOutputPath;@@ -208,27 +210,27 @@ class SnapshotHelper {log: string[] | undefined,step: TestStepInfoImpl | undefined): ImageMatcherResult {const output = [`${header}${indent(diffError, ' ')}`];+ if (this.name) {+ output.push('');+ output.push(` Snapshot: ${this.name}`);+ }if (expected !== undefined) {// Copy the expectation inside the `test-results/` folder for backwards compatibility,// so that one can upload `test-results/` directory and have all the data inside.writeFileSync(this.legacyExpectedPath, expected);step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath });- output.push(`\nExpected: ${colors.yellow(this.expectedPath)}`);}if (previous !== undefined) {writeFileSync(this.previousPath, previous);step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath });- output.push(`Previous: ${colors.yellow(this.previousPath)}`);}if (actual !== undefined) {writeFileSync(this.actualPath, actual);step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath });- output.push(`Received: ${colors.yellow(this.actualPath)}`);}if (diff !== undefined) {writeFileSync(this.diffPath, diff);step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath });- output.push(` Diff: ${colors.yellow(this.diffPath)}`);}if (log?.length)diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex ee8640957f..5a3e033ef8 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -354,29 +354,49 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCaseresultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));}resultLines.push(...errors.map(error => '\n' + error.message));- for (let i = 0; i < result.attachments.length; ++i) {- const attachment = result.attachments[i];+ const attachmentGroups = groupAttachments(result.attachments);+ for (let i = 0; i < attachmentGroups.length; ++i) {+ const attachment = attachmentGroups[i];if (attachment.name === 'error-context' && attachment.path) {resultLines.push('');resultLines.push(screen.colors.dim(` Error Context: ${relativeFilePath(screen, config, attachment.path)}`));continue;}+if (attachment.name.startsWith('_'))continue;+const hasPrintableContent = attachment.contentType.startsWith('text/');if (!attachment.path && !hasPrintableContent)continue;+resultLines.push('');- resultLines.push(screen.colors.cyan(separator(screen, ` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`)));- if (attachment.path) {- const relativePath = path.relative(process.cwd(), attachment.path);- resultLines.push(screen.colors.cyan(` ${relativePath}`));+ resultLines.push(screen.colors.dim(separator(screen, ` attachment #${i + 1}: ${screen.colors.bold(attachment.name)} (${attachment.contentType})`)));++ if (attachment.actual?.path) {+ if (attachment.expected?.path) {+ const expectedPath = relativeFilePath(screen, config, attachment.expected.path);+ resultLines.push(screen.colors.dim(` Expected: ${expectedPath}`));+ }+ const actualPath = relativeFilePath(screen, config, attachment.actual.path);+ resultLines.push(screen.colors.dim(` Received: ${actualPath}`));+ if (attachment.previous?.path) {+ const previousPath = relativeFilePath(screen, config, attachment.previous.path);+ resultLines.push(screen.colors.dim(` Previous: ${previousPath}`));+ }+ if (attachment.diff?.path) {+ const diffPath = relativeFilePath(screen, config, attachment.diff.path);+ resultLines.push(screen.colors.dim(` Diff: ${diffPath}`));+ }+ } else if (attachment.path) {+ const relativePath = relativeFilePath(screen, config, attachment.path);+ resultLines.push(screen.colors.dim(` ${relativePath}`));// Make this extensibleif (attachment.name === 'trace') {const packageManagerCommand = getPackageManagerExecCommand();- resultLines.push(screen.colors.cyan(` Usage:`));+ resultLines.push(screen.colors.dim(` Usage:`));resultLines.push('');- resultLines.push(screen.colors.cyan(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`));+ resultLines.push(screen.colors.dim(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`));resultLines.push('');}} else {@@ -385,10 +405,10 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCaseif (text.length > 300)text = text.slice(0, 300) + '...';for (const line of text.split('\n'))- resultLines.push(screen.colors.cyan(` ${line}`));+ resultLines.push(screen.colors.dim(` ${line}`));}}- resultLines.push(screen.colors.cyan(separator(screen, ' ')));+ resultLines.push(screen.colors.dim(separator(screen, ' ')));}lines.push(...resultLines);}@@ -532,7 +552,7 @@ export function separator(screen: Screen, text: string = ''): string {if (text)text += ' ';const columns = Math.min(100, screen.ttyWidth || 100);- return text + screen.colors.dim('─'.repeat(Math.max(0, columns - text.length)));+ return text + screen.colors.dim('─'.repeat(Math.max(0, columns - stripAnsiEscapes(text).length)));}function indent(lines: string, tab: string) {@@ -640,3 +660,46 @@ export function resolveOutputFile(reporterName: string, options: {return { outputFile, outputDir };}++type TestAttachment = TestResult['attachments'][number];++type TestAttachmentGroup = TestAttachment & {+ expected?: TestAttachment;+ actual?: TestAttachment;+ diff?: TestAttachment;+ previous?: TestAttachment;+};++function groupAttachments(attachments: TestResult['attachments']): TestAttachmentGroup[] {+ const result: TestAttachmentGroup[] = [];+ const attachmentsByPrefix = new Map<string, TestAttachment>();+ for (const attachment of attachments) {+ if (!attachment.path) {+ result.push(attachment);+ continue;+ }++ const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/);+ if (!match) {+ result.push(attachment);+ continue;+ }++ const [, name, category] = match;+ let group: TestAttachmentGroup | undefined = attachmentsByPrefix.get(name);+ if (!group) {+ group = { ...attachment, name };+ attachmentsByPrefix.set(name, group);+ result.push(group);+ }+ if (category === 'expected')+ group.expected = attachment;+ else if (category === 'actual')+ group.actual = attachment;+ else if (category === 'diff')+ group.diff = attachment;+ else if (category === 'previous')+ group.previous = attachment;+ }+ return result;+}diff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.tsindex 676a29fc67..81feca0b48 100644--- a/tests/page/expect-matcher-result.spec.ts+++ b/tests/page/expect-matcher-result.spec.ts@@ -298,6 +298,5 @@ test('toHaveScreenshot should populate matcherResult', async ({ page, server, isexpect.soft(stripAnsi(e.toString())).toContain(`Error: expect(page).toHaveScreenshot(expected)23362 pixels (ratio 0.10 of all image pixels) are different.--Expected:`);+`);});diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.tsindex da6c2a8022..7d59e7c3c0 100644--- a/tests/playwright-test/golden.spec.ts+++ b/tests/playwright-test/golden.spec.ts@@ -228,7 +228,7 @@ test('should write detailed failure result to an output folder', async ({ runInlconst expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt');const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt');expect(outputText).toMatch(/Expected:.*a\.spec\.js-snapshots.snapshot\.txt/);- expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);+ expect(outputText).toContain(`Received: test-results${path.sep}a-is-a-test${path.sep}snapshot-actual.txt`);expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);});@@ -672,8 +672,8 @@ test('should compare different PNG images', async ({ runInlineTest }, testInfo)const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png');const diffSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-diff.png');expect(outputText).toMatch(/Expected:.*a\.spec\.js-snapshots.snapshot\.png/);- expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);- expect(outputText).toContain(`Diff: ${diffSnapshotArtifactPath}`);+ expect(outputText).toContain(`Received: test-results${path.sep}a-is-a-test${path.sep}snapshot-actual.png`);+ expect(outputText).toContain(`Diff: test-results${path.sep}a-is-a-test${path.sep}snapshot-diff.png`);expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);expect(fs.existsSync(diffSnapshotArtifactPath)).toBe(true);diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex 96c7d916ae..e4194be748 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -289,11 +289,9 @@ for (const useIntermediateMergeReport of [true, false] as const) {await showReport();await page.getByRole('link', { name: 'fails' }).click();- await expect(page.getByTestId('test-screenshot-error-view').getByTestId('error-suffix')).toContainText([- `> 6 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,- `> 7 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,- `> 8 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,- ]);+ await expect(page.locator('.test-error-view').first()).toContainText(+ `> 6 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,+ );const imageDiffs = page.getByTestId('test-results-image-diff');await expect(imageDiffs.getByTestId('test-result-image-mismatch')).toHaveCount(3);await expect(imageDiffs.getByText('Image mismatch:')).toHaveText([@@ -303,44 +301,6 @@ for (const useIntermediateMergeReport of [true, false] as const) {]);});- test('should include image diff when screenshot failed to generate due to animation', async ({ runInlineTest, page, showReport }) => {- test.skip(process.env.PW_CLOCK === 'frozen', 'Assumes Date.now() changes');- const result = await runInlineTest({- 'playwright.config.ts': `- module.exports = { use: { viewport: { width: 200, height: 200 }} };- `,- 'a.test.js': `- import { test, expect } from '@playwright/test';- test('fails', async ({ page }, testInfo) => {- testInfo.snapshotSuffix = '';- await page.evaluate(() => {- setInterval(() => {- document.body.textContent = Date.now();- }, 50);- });- await expect.soft(page).toHaveScreenshot({ timeout: 1000 });- });- `,- }, { 'reporter': 'dot,html', 'update-snapshots': true }, { PLAYWRIGHT_HTML_OPEN: 'never' });- expect(result.exitCode).toBe(1);- expect(result.failed).toBe(1);-- await showReport();- await page.getByRole('link', { name: 'fails' }).click();- await expect(page.locator('text=Image mismatch')).toHaveCount(1);- await expect(page.locator('text=Snapshot mismatch')).toHaveCount(0);- await expect(page.locator('.chip-header', { hasText: 'Screenshots' })).toHaveCount(0);- const errorChip = page.getByTestId('test-screenshot-error-view');- await expect(errorChip).toContainText('Failed to take two consecutive stable screenshots.');- await expect(errorChip.getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([- 'Diff',- 'Actual',- 'Previous',- 'Side by side',- 'Slider',- ]);- });-test('should not include image diff with non-images', async ({ runInlineTest, page, showReport }) => {const expected = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAAAXNSR0IArs4c6QAAAhVJREFUeJzt07ERwCAQwLCQ/Xd+FuDcQiFN4MZrZuYDjv7bAfAyg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAiEDVPZBYx6ffy+AAAAAElFTkSuQmCC', 'base64');const result = await runInlineTest({diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.tsindex 330a7991d9..df46e9c2bc 100644--- a/tests/playwright-test/to-have-screenshot.spec.ts+++ b/tests/playwright-test/to-have-screenshot.spec.ts@@ -157,6 +157,8 @@ test.describe('expect config animations option', () => {`}, { 'update-snapshots': true });expect(result.exitCode).toBe(1);+ expect(result.output).toContain('is-a-test-1-actual.png');+ expect(result.output).toContain('is-a-test-1-previous.png');expect(result.output).toContain('is-a-test-1-diff.png');});});