Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: group attachments in terminal reporters (#36607)

Pavel Feldman2cbe965880ba
diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx
index 7639e3577b..6fa7bc50ec 100644
--- a/packages/html-reporter/src/testErrorView.tsx
+++ b/packages/html-reporter/src/testErrorView.tsx
@@ -47,16 +47,10 @@ export const PromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {
};
export const TestScreenshotErrorView: React.FC<{
- errorPrefix?: string,
diff: ImageDiff,
- errorSuffix?: string,
-}> = ({ errorPrefix, diff, errorSuffix }) => {
- const prefixHtml = React.useMemo(() => ansiErrorToHtml(errorPrefix), [errorPrefix]);
- const suffixHtml = React.useMemo(() => ansiErrorToHtml(errorSuffix), [errorSuffix]);
+}> = ({ diff }) => {
return <div data-testid='test-screenshot-error-view' className='test-error-view'>
- <div dangerouslySetInnerHTML={{ __html: prefixHtml || '' }} className='test-error-text' style={{ marginBottom: 20 }}></div>
<ImageDiffView key='image-diff' diff={diff} hideDetails={true}></ImageDiffView>
- <div data-testid='error-suffix' dangerouslySetInnerHTML={{ __html: suffixHtml || '' }} className='test-error-text'></div>
</div>;
};
diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx
index b023c27981..c8f8f04029 100644
--- a/packages/html-reporter/src/testResultView.tsx
+++ b/packages/html-reporter/src/testResultView.tsx
@@ -86,7 +86,7 @@ export const TestResultView: React.FC<{
[...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));
const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`);
const diffs = groupImageDiffs(screenshots, result);
- const errors = classifyErrors(result.errors.map(e => e.message), diffs);
+ const errors = result.errors.map(e => e.message);
return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext };
}, [result]);
@@ -111,9 +111,11 @@ export const TestResultView: React.FC<{
</div>
)}
{errors.map((error, index) => {
- if (error.type === 'screenshot')
- return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;
- return <CodeSnippet key={'test-result-error-message-' + index} code={error.error!}/>;
+ const diff = pickDiffForError(error, diffs);
+ return <>
+ <CodeSnippet key={'test-result-error-message-' + index} code={error}/>
+ {diff && <TestScreenshotErrorView diff={diff}></TestScreenshotErrorView>}
+ </>;
})}
</AutoChip>}
{!!result.steps.length && <AutoChip header='Test Steps'>
@@ -167,29 +169,11 @@ export const TestResultView: React.FC<{
</div>;
};
-function classifyErrors(testErrors: string[], diffs: ImageDiff[]) {
- return testErrors.map(error => {
- const firstLine = error.split('\n')[0];
- if (firstLine.includes('toHaveScreenshot') || firstLine.includes('toMatchSnapshot')) {
- const matchingDiff = diffs.find(diff => {
- const attachmentName = diff.actual?.attachment.name;
- return attachmentName && error.includes(attachmentName);
- });
-
- if (matchingDiff) {
- const lines = error.split('\n');
- const index = lines.findIndex(line => /Expected:|Previous:|Received:/.test(line));
- const errorPrefix = index !== -1 ? lines.slice(0, index).join('\n') : lines[0];
-
- const diffIndex = lines.findIndex(line => / +Diff:/.test(line));
- const errorSuffix = diffIndex !== -1 ? lines.slice(diffIndex + 2).join('\n') : lines.slice(1).join('\n');
-
- return { type: 'screenshot', diff: matchingDiff, errorPrefix, errorSuffix };
- }
- }
-
- return { type: 'regular', error };
- });
+function pickDiffForError(error: string, diffs: ImageDiff[]): ImageDiff | undefined {
+ const firstLine = error.split('\n')[0];
+ if (!firstLine.includes('toHaveScreenshot') && !firstLine.includes('toMatchSnapshot'))
+ return undefined;
+ return diffs.find(diff => error.includes(diff.name));
}
const StepTreeItem: React.FC<{
diff --git a/packages/playwright/src/matchers/toMatchSnapshot.ts b/packages/playwright/src/matchers/toMatchSnapshot.ts
index b2ef3c3323..ac3e5344b3 100644
--- a/packages/playwright/src/matchers/toMatchSnapshot.ts
+++ b/packages/playwright/src/matchers/toMatchSnapshot.ts
@@ -68,6 +68,7 @@ const NonConfigProperties: (keyof ToHaveScreenshotOptions)[] = [
class SnapshotHelper {
readonly testInfo: TestInfoImpl;
+ readonly name: string;
readonly attachmentBaseName: string;
readonly legacyExpectedPath: string;
readonly previousPath: string;
@@ -101,6 +102,7 @@ class SnapshotHelper {
name = nameFromOptions;
}
+ this.name = Array.isArray(name) ? name.join(path.sep) : name || '';
const resolvedPaths = testInfo._resolveSnapshotPaths(matcherName === 'toHaveScreenshot' ? 'screenshot' : 'snapshot', name, 'updateSnapshotIndex', anonymousSnapshotExtension);
this.expectedPath = resolvedPaths.absoluteSnapshotPath;
this.attachmentBaseName = resolvedPaths.relativeOutputPath;
@@ -208,27 +210,27 @@ class SnapshotHelper {
log: string[] | undefined,
step: TestStepInfoImpl | undefined): ImageMatcherResult {
const output = [`${header}${indent(diffError, ' ')}`];
+ if (this.name) {
+ output.push('');
+ output.push(` Snapshot: ${this.name}`);
+ }
if (expected !== undefined) {
// Copy the expectation inside the `test-results/` folder for backwards compatibility,
// so that one can upload `test-results/` directory and have all the data inside.
writeFileSync(this.legacyExpectedPath, expected);
step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath });
- output.push(`\nExpected: ${colors.yellow(this.expectedPath)}`);
}
if (previous !== undefined) {
writeFileSync(this.previousPath, previous);
step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-previous'), contentType: this.mimeType, path: this.previousPath });
- output.push(`Previous: ${colors.yellow(this.previousPath)}`);
}
if (actual !== undefined) {
writeFileSync(this.actualPath, actual);
step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath });
- output.push(`Received: ${colors.yellow(this.actualPath)}`);
}
if (diff !== undefined) {
writeFileSync(this.diffPath, diff);
step?._attachToStep({ name: addSuffixToFilePath(this.attachmentBaseName, '-diff'), contentType: this.mimeType, path: this.diffPath });
- output.push(` Diff: ${colors.yellow(this.diffPath)}`);
}
if (log?.length)
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index ee8640957f..5a3e033ef8 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -354,29 +354,49 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase
resultLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`)));
}
resultLines.push(...errors.map(error => '\n' + error.message));
- for (let i = 0; i < result.attachments.length; ++i) {
- const attachment = result.attachments[i];
+ const attachmentGroups = groupAttachments(result.attachments);
+ for (let i = 0; i < attachmentGroups.length; ++i) {
+ const attachment = attachmentGroups[i];
if (attachment.name === 'error-context' && attachment.path) {
resultLines.push('');
resultLines.push(screen.colors.dim(` Error Context: ${relativeFilePath(screen, config, attachment.path)}`));
continue;
}
+
if (attachment.name.startsWith('_'))
continue;
+
const hasPrintableContent = attachment.contentType.startsWith('text/');
if (!attachment.path && !hasPrintableContent)
continue;
+
resultLines.push('');
- resultLines.push(screen.colors.cyan(separator(screen, ` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`)));
- if (attachment.path) {
- const relativePath = path.relative(process.cwd(), attachment.path);
- resultLines.push(screen.colors.cyan(` ${relativePath}`));
+ resultLines.push(screen.colors.dim(separator(screen, ` attachment #${i + 1}: ${screen.colors.bold(attachment.name)} (${attachment.contentType})`)));
+
+ if (attachment.actual?.path) {
+ if (attachment.expected?.path) {
+ const expectedPath = relativeFilePath(screen, config, attachment.expected.path);
+ resultLines.push(screen.colors.dim(` Expected: ${expectedPath}`));
+ }
+ const actualPath = relativeFilePath(screen, config, attachment.actual.path);
+ resultLines.push(screen.colors.dim(` Received: ${actualPath}`));
+ if (attachment.previous?.path) {
+ const previousPath = relativeFilePath(screen, config, attachment.previous.path);
+ resultLines.push(screen.colors.dim(` Previous: ${previousPath}`));
+ }
+ if (attachment.diff?.path) {
+ const diffPath = relativeFilePath(screen, config, attachment.diff.path);
+ resultLines.push(screen.colors.dim(` Diff: ${diffPath}`));
+ }
+ } else if (attachment.path) {
+ const relativePath = relativeFilePath(screen, config, attachment.path);
+ resultLines.push(screen.colors.dim(` ${relativePath}`));
// Make this extensible
if (attachment.name === 'trace') {
const packageManagerCommand = getPackageManagerExecCommand();
- resultLines.push(screen.colors.cyan(` Usage:`));
+ resultLines.push(screen.colors.dim(` Usage:`));
resultLines.push('');
- resultLines.push(screen.colors.cyan(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`));
+ resultLines.push(screen.colors.dim(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`));
resultLines.push('');
}
} else {
@@ -385,10 +405,10 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase
if (text.length > 300)
text = text.slice(0, 300) + '...';
for (const line of text.split('\n'))
- resultLines.push(screen.colors.cyan(` ${line}`));
+ resultLines.push(screen.colors.dim(` ${line}`));
}
}
- resultLines.push(screen.colors.cyan(separator(screen, ' ')));
+ resultLines.push(screen.colors.dim(separator(screen, ' ')));
}
lines.push(...resultLines);
}
@@ -532,7 +552,7 @@ export function separator(screen: Screen, text: string = ''): string {
if (text)
text += ' ';
const columns = Math.min(100, screen.ttyWidth || 100);
- return text + screen.colors.dim('─'.repeat(Math.max(0, columns - text.length)));
+ return text + screen.colors.dim('─'.repeat(Math.max(0, columns - stripAnsiEscapes(text).length)));
}
function indent(lines: string, tab: string) {
@@ -640,3 +660,46 @@ export function resolveOutputFile(reporterName: string, options: {
return { outputFile, outputDir };
}
+
+type TestAttachment = TestResult['attachments'][number];
+
+type TestAttachmentGroup = TestAttachment & {
+ expected?: TestAttachment;
+ actual?: TestAttachment;
+ diff?: TestAttachment;
+ previous?: TestAttachment;
+};
+
+function groupAttachments(attachments: TestResult['attachments']): TestAttachmentGroup[] {
+ const result: TestAttachmentGroup[] = [];
+ const attachmentsByPrefix = new Map<string, TestAttachment>();
+ for (const attachment of attachments) {
+ if (!attachment.path) {
+ result.push(attachment);
+ continue;
+ }
+
+ const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/);
+ if (!match) {
+ result.push(attachment);
+ continue;
+ }
+
+ const [, name, category] = match;
+ let group: TestAttachmentGroup | undefined = attachmentsByPrefix.get(name);
+ if (!group) {
+ group = { ...attachment, name };
+ attachmentsByPrefix.set(name, group);
+ result.push(group);
+ }
+ if (category === 'expected')
+ group.expected = attachment;
+ else if (category === 'actual')
+ group.actual = attachment;
+ else if (category === 'diff')
+ group.diff = attachment;
+ else if (category === 'previous')
+ group.previous = attachment;
+ }
+ return result;
+}
diff --git a/tests/page/expect-matcher-result.spec.ts b/tests/page/expect-matcher-result.spec.ts
index 676a29fc67..81feca0b48 100644
--- a/tests/page/expect-matcher-result.spec.ts
+++ b/tests/page/expect-matcher-result.spec.ts
@@ -298,6 +298,5 @@ test('toHaveScreenshot should populate matcherResult', async ({ page, server, is
expect.soft(stripAnsi(e.toString())).toContain(`Error: expect(page).toHaveScreenshot(expected)
23362 pixels (ratio 0.10 of all image pixels) are different.
-
-Expected:`);
+`);
});
diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts
index da6c2a8022..7d59e7c3c0 100644
--- a/tests/playwright-test/golden.spec.ts
+++ b/tests/playwright-test/golden.spec.ts
@@ -228,7 +228,7 @@ test('should write detailed failure result to an output folder', async ({ runInl
const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.txt');
const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.txt');
expect(outputText).toMatch(/Expected:.*a\.spec\.js-snapshots.snapshot\.txt/);
- expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);
+ expect(outputText).toContain(`Received: test-results${path.sep}a-is-a-test${path.sep}snapshot-actual.txt`);
expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);
expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);
});
@@ -672,8 +672,8 @@ test('should compare different PNG images', async ({ runInlineTest }, testInfo)
const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png');
const diffSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-diff.png');
expect(outputText).toMatch(/Expected:.*a\.spec\.js-snapshots.snapshot\.png/);
- expect(outputText).toContain(`Received: ${actualSnapshotArtifactPath}`);
- expect(outputText).toContain(`Diff: ${diffSnapshotArtifactPath}`);
+ expect(outputText).toContain(`Received: test-results${path.sep}a-is-a-test${path.sep}snapshot-actual.png`);
+ expect(outputText).toContain(`Diff: test-results${path.sep}a-is-a-test${path.sep}snapshot-diff.png`);
expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(true);
expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(true);
expect(fs.existsSync(diffSnapshotArtifactPath)).toBe(true);
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index 96c7d916ae..e4194be748 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -289,11 +289,9 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await showReport();
await page.getByRole('link', { name: 'fails' }).click();
- await expect(page.getByTestId('test-screenshot-error-view').getByTestId('error-suffix')).toContainText([
- `> 6 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,
- `> 7 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,
- `> 8 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,
- ]);
+ await expect(page.locator('.test-error-view').first()).toContainText(
+ `> 6 | await expect.soft(screenshot).toMatchSnapshot('expected.png');`,
+ );
const imageDiffs = page.getByTestId('test-results-image-diff');
await expect(imageDiffs.getByTestId('test-result-image-mismatch')).toHaveCount(3);
await expect(imageDiffs.getByText('Image mismatch:')).toHaveText([
@@ -303,44 +301,6 @@ for (const useIntermediateMergeReport of [true, false] as const) {
]);
});
- test('should include image diff when screenshot failed to generate due to animation', async ({ runInlineTest, page, showReport }) => {
- test.skip(process.env.PW_CLOCK === 'frozen', 'Assumes Date.now() changes');
- const result = await runInlineTest({
- 'playwright.config.ts': `
- module.exports = { use: { viewport: { width: 200, height: 200 }} };
- `,
- 'a.test.js': `
- import { test, expect } from '@playwright/test';
- test('fails', async ({ page }, testInfo) => {
- testInfo.snapshotSuffix = '';
- await page.evaluate(() => {
- setInterval(() => {
- document.body.textContent = Date.now();
- }, 50);
- });
- await expect.soft(page).toHaveScreenshot({ timeout: 1000 });
- });
- `,
- }, { 'reporter': 'dot,html', 'update-snapshots': true }, { PLAYWRIGHT_HTML_OPEN: 'never' });
- expect(result.exitCode).toBe(1);
- expect(result.failed).toBe(1);
-
- await showReport();
- await page.getByRole('link', { name: 'fails' }).click();
- await expect(page.locator('text=Image mismatch')).toHaveCount(1);
- await expect(page.locator('text=Snapshot mismatch')).toHaveCount(0);
- await expect(page.locator('.chip-header', { hasText: 'Screenshots' })).toHaveCount(0);
- const errorChip = page.getByTestId('test-screenshot-error-view');
- await expect(errorChip).toContainText('Failed to take two consecutive stable screenshots.');
- await expect(errorChip.getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([
- 'Diff',
- 'Actual',
- 'Previous',
- 'Side by side',
- 'Slider',
- ]);
- });
-
test('should not include image diff with non-images', async ({ runInlineTest, page, showReport }) => {
const expected = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAAAXNSR0IArs4c6QAAAhVJREFUeJzt07ERwCAQwLCQ/Xd+FuDcQiFN4MZrZuYDjv7bAfAyg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAgEg0AwCASDQDAIBINAMAiEDVPZBYx6ffy+AAAAAElFTkSuQmCC', 'base64');
const result = await runInlineTest({
diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts
index 330a7991d9..df46e9c2bc 100644
--- a/tests/playwright-test/to-have-screenshot.spec.ts
+++ b/tests/playwright-test/to-have-screenshot.spec.ts
@@ -157,6 +157,8 @@ test.describe('expect config animations option', () => {
`
}, { 'update-snapshots': true });
expect(result.exitCode).toBe(1);
+ expect(result.output).toContain('is-a-test-1-actual.png');
+ expect(result.output).toContain('is-a-test-1-previous.png');
expect(result.output).toContain('is-a-test-1-diff.png');
});
});