Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(list): print only the last attempt for inline failures with retries (#42648)

Pavel Feldman37f7d8f9809a
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index cd49a6eb71..062584f54d 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -164,6 +164,7 @@ export type TerminalReporterOptions = {
omitFailures?: boolean;
includeTestId?: boolean;
omitTags?: boolean;
+ lastResult?: boolean;
};
export class TerminalReporter implements ReporterV2 {
@@ -402,7 +403,8 @@ function formatResultErrors(screen: Screen, test: TestCase, result: TestResult):
export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {
const lines: string[] = [];
let printedHeader = false;
- for (const result of test.results) {
+ const results = options?.lastResult ? test.results.slice(-1) : test.results;
+ for (const result of results) {
const resultLines: string[] = [];
const errors = formatResultFailure(screen, test, result, ' ');
if (!errors.length)
diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts
index eb128a33db..68db5a0eca 100644
--- a/packages/playwright/src/reporters/list.ts
+++ b/packages/playwright/src/reporters/list.ts
@@ -39,13 +39,14 @@ class ListReporter extends TerminalReporter {
private _needNewLine = false;
private _printSteps: boolean;
private _printFailuresInline: boolean;
- private _failureIndex = 0;
+ private _failureIndex = new Map<TestCase, number>();
private _paused = new Set<TestResult>();
constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {
- super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags) });
+ const printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline);
+ super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags), lastResult: printFailuresInline });
this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);
- this._printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline);
+ this._printFailuresInline = printFailuresInline;
}
override onBegin(suite: Suite) {
@@ -201,7 +202,13 @@ class ListReporter extends TerminalReporter {
private _printFailure(test: TestCase) {
this._maybeWriteNewLine();
- const message = '\n' + this.formatFailure(test, ++this._failureIndex) + '\n';
+ // Retries of the same test share one failure index.
+ let index = this._failureIndex.get(test);
+ if (index === undefined) {
+ index = this._failureIndex.size + 1;
+ this._failureIndex.set(test, index);
+ }
+ const message = '\n' + this.formatFailure(test, index) + '\n';
this._updateLineCountAndNewLineFlagForOutput(message);
this.screen.stdout.write(message);
}
diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts
index fe333c6e12..dcda138cb8 100644
--- a/tests/playwright-test/reporter-list.spec.ts
+++ b/tests/playwright-test/reporter-list.spec.ts
@@ -316,6 +316,63 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(result.exitCode).toBe(1);
});
+ test('print failures inline with retries', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: [['list', { printFailuresInline: true }]],
+ retries: 2,
+ workers: 1,
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('fails', async ({}, testInfo) => {
+ throw new Error('attempt ' + testInfo.retry);
+ });
+ `,
+ });
+ const text = result.output;
+ const failureHeader = '1) a.test.ts:3:15 › fails';
+ // Each attempt is printed once, under the same failure index.
+ expect(text.split(failureHeader).length - 1).toBe(3);
+ expect(text).not.toContain('2) a.test.ts:3:15 › fails');
+ for (const attempt of ['attempt 0', 'attempt 1', 'attempt 2'])
+ expect(text.split('Error: ' + attempt).length - 1).toBe(1);
+ expect(text.split('Retry #1').length - 1).toBe(1);
+ expect(text.split('Retry #2').length - 1).toBe(1);
+ // Retry sections come after the corresponding retry attempt.
+ expect(text.indexOf('Retry #1')).toBeGreaterThan(text.indexOf('Error: attempt 0'));
+ expect(text.indexOf('Retry #2')).toBeGreaterThan(text.indexOf('Error: attempt 1'));
+ expect(result.exitCode).toBe(1);
+ });
+
+ test('print failures inline for a flaky test', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ reporter: [['list', { printFailuresInline: true }]],
+ retries: 1,
+ workers: 1,
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('flaky', async ({}, testInfo) => {
+ if (testInfo.retry === 0)
+ throw new Error('attempt ' + testInfo.retry);
+ });
+ `,
+ });
+ const text = result.output;
+ const failureHeader = '1) a.test.ts:3:15 › flaky';
+ expect(text.split(failureHeader).length - 1).toBe(1);
+ expect(text.split('Error: attempt 0').length - 1).toBe(1);
+ expect(text).not.toContain('Retry #1');
+ expect(text).toContain('1 flaky');
+ expect(result.exitCode).toBe(0);
+ });
+
test('print stdio', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `