Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(list): print only the last attempt for inline failures with retries (#42648)
Pavel Feldman37f7d8f9809a
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex cd49a6eb71..062584f54d 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -164,6 +164,7 @@ export type TerminalReporterOptions = {omitFailures?: boolean;includeTestId?: boolean;omitTags?: boolean;+ lastResult?: boolean;};export class TerminalReporter implements ReporterV2 {@@ -402,7 +403,8 @@ function formatResultErrors(screen: Screen, test: TestCase, result: TestResult):export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {const lines: string[] = [];let printedHeader = false;- for (const result of test.results) {+ const results = options?.lastResult ? test.results.slice(-1) : test.results;+ for (const result of results) {const resultLines: string[] = [];const errors = formatResultFailure(screen, test, result, ' ');if (!errors.length)diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.tsindex eb128a33db..68db5a0eca 100644--- a/packages/playwright/src/reporters/list.ts+++ b/packages/playwright/src/reporters/list.ts@@ -39,13 +39,14 @@ class ListReporter extends TerminalReporter {private _needNewLine = false;private _printSteps: boolean;private _printFailuresInline: boolean;- private _failureIndex = 0;+ private _failureIndex = new Map<TestCase, number>();private _paused = new Set<TestResult>();constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {- super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags) });+ const printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline);+ super({ ...options, omitTags: getAsBooleanFromENV('PLAYWRIGHT_LIST_OMIT_TAGS', options?.omitTags), lastResult: printFailuresInline });this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);- this._printFailuresInline = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_FAILURES_INLINE', options?.printFailuresInline);+ this._printFailuresInline = printFailuresInline;}override onBegin(suite: Suite) {@@ -201,7 +202,13 @@ class ListReporter extends TerminalReporter {private _printFailure(test: TestCase) {this._maybeWriteNewLine();- const message = '\n' + this.formatFailure(test, ++this._failureIndex) + '\n';+ // Retries of the same test share one failure index.+ let index = this._failureIndex.get(test);+ if (index === undefined) {+ index = this._failureIndex.size + 1;+ this._failureIndex.set(test, index);+ }+ const message = '\n' + this.formatFailure(test, index) + '\n';this._updateLineCountAndNewLineFlagForOutput(message);this.screen.stdout.write(message);}diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.tsindex fe333c6e12..dcda138cb8 100644--- a/tests/playwright-test/reporter-list.spec.ts+++ b/tests/playwright-test/reporter-list.spec.ts@@ -316,6 +316,63 @@ for (const useIntermediateMergeReport of [false, true] as const) {expect(result.exitCode).toBe(1);});+ test('print failures inline with retries', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ reporter: [['list', { printFailuresInline: true }]],+ retries: 2,+ workers: 1,+ };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('fails', async ({}, testInfo) => {+ throw new Error('attempt ' + testInfo.retry);+ });+ `,+ });+ const text = result.output;+ const failureHeader = '1) a.test.ts:3:15 › fails';+ // Each attempt is printed once, under the same failure index.+ expect(text.split(failureHeader).length - 1).toBe(3);+ expect(text).not.toContain('2) a.test.ts:3:15 › fails');+ for (const attempt of ['attempt 0', 'attempt 1', 'attempt 2'])+ expect(text.split('Error: ' + attempt).length - 1).toBe(1);+ expect(text.split('Retry #1').length - 1).toBe(1);+ expect(text.split('Retry #2').length - 1).toBe(1);+ // Retry sections come after the corresponding retry attempt.+ expect(text.indexOf('Retry #1')).toBeGreaterThan(text.indexOf('Error: attempt 0'));+ expect(text.indexOf('Retry #2')).toBeGreaterThan(text.indexOf('Error: attempt 1'));+ expect(result.exitCode).toBe(1);+ });++ test('print failures inline for a flaky test', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ reporter: [['list', { printFailuresInline: true }]],+ retries: 1,+ workers: 1,+ };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('flaky', async ({}, testInfo) => {+ if (testInfo.retry === 0)+ throw new Error('attempt ' + testInfo.retry);+ });+ `,+ });+ const text = result.output;+ const failureHeader = '1) a.test.ts:3:15 › flaky';+ expect(text.split(failureHeader).length - 1).toBe(1);+ expect(text.split('Error: attempt 0').length - 1).toBe(1);+ expect(text).not.toContain('Retry #1');+ expect(text).toContain('1 flaky');+ expect(result.exitCode).toBe(0);+ });+test('print stdio', async ({ runInlineTest }) => {const result = await runInlineTest({'a.test.ts': `