Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(junit): add includeRetries option to emit each retry as separate testcase (#39464)
vazidmansuri0050154add954e0
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.tsindex 6b2c7561dc..80c1548474 100644--- a/packages/playwright/src/reporters/junit.ts+++ b/packages/playwright/src/reporters/junit.ts@@ -24,7 +24,7 @@ import { stripAnsiEscapes } from '../util';import type { ReporterV2 } from './reporterV2';import type { JUnitReporterOptions } from '../../types/test';-import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter';+import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter';class JUnitReporter implements ReporterV2 {private config!: FullConfig;@@ -38,10 +38,12 @@ class JUnitReporter implements ReporterV2 {private resolvedOutputFile: string | undefined;private stripANSIControlSequences = false;private includeProjectInTestName = false;+ private includeRetries = false;constructor(options: JUnitReporterOptions & CommonReporterOptions) {this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences);this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName);+ this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries);this.configDir = options.configDir;this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile;}@@ -143,6 +145,9 @@ class JUnitReporter implements ReporterV2 {}private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> {+ const isRetried = this.includeRetries && test.results.length > 1;+ const isFlaky = isRetried && test.ok();+const entry = {name: 'testcase',attributes: {@@ -150,7 +155,14 @@ class JUnitReporter implements ReporterV2 {name: namePrefix + test.titlePath().slice(3).join(' › '),// filenameclassname: suiteName,- time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000+ // For flaky tests, use the last (successful) result's duration.+ // For permanent failures with retries, use the first result's duration.+ // Otherwise, use total duration across all results.+ time: isFlaky+ ? test.results[test.results.length - 1].duration / 1000+ : isRetried+ ? test.results[0].duration / 1000+ : (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000},children: [] as XMLEntry[]@@ -185,34 +197,40 @@ class JUnitReporter implements ReporterV2 {}let classification: 'failure' | 'error' | null = null;- if (!test.ok()) {- const errorInfo = classifyError(test);- if (errorInfo) {- classification = errorInfo.elementName;- entry.children.push({- name: errorInfo.elementName,- attributes: {- message: errorInfo.message,- type: errorInfo.type,- },- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))- });- } else {- classification = 'failure';- entry.children.push({- name: 'failure',- attributes: {- message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,- type: 'FAILURE',- },- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))- });++ if (isFlaky) {+ // Flaky test (eventually passed): use Maven Surefire <flakyFailure>/<flakyError>.+ // No <failure> element — flaky tests count as passed.+ for (const result of test.results) {+ if (result.status === 'passed' || result.status === 'skipped')+ continue;+ entry.children.push(buildSurefireRetryEntry(result, 'flaky'));}+ // classification stays null — flaky tests are not counted as failures.+ } else if (isRetried) {+ // Permanent failure (failed all retries): use <failure> + Maven Surefire <rerunFailure>/<rerunError>.+ classification = this._addFailureEntry(test, entry);+ // Add <rerunFailure>/<rerunError> for each subsequent retry.+ for (let i = 1; i < test.results.length; i++) {+ const result = test.results[i];+ if (result.status === 'passed' || result.status === 'skipped')+ continue;+ entry.children.push(buildSurefireRetryEntry(result, 'rerun'));+ }+ } else if (!test.ok()) {+ // Standard failure (no retries, or includeRetries is false).+ classification = this._addFailureEntry(test, entry);}const systemOut: string[] = [];const systemErr: string[] = [];- for (const result of test.results) {+ // When retries are included, top-level output comes from the primary result only:+ // flaky → last (successful) result; permanent failure → first result.+ // Without retries: all results (original behavior).+ const outputResults = isRetried+ ? [isFlaky ? test.results[test.results.length - 1] : test.results[0]]+ : test.results;+ for (const result of outputResults) {for (const item of result.stdout)systemOut.push(item.toString());for (const item of result.stderr)@@ -245,40 +263,92 @@ class JUnitReporter implements ReporterV2 {entry.children.push({ name: 'system-err', text: systemErr.join('') });return classification;}-}-function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {- for (const result of test.results) {- const error = result.error;- if (!error)- continue;-- const rawMessage = stripAnsiEscapes(error.message || error.value || '');-- // Parse "ErrorName: message" format from serialized error.- const nameMatch = rawMessage.match(/^(\w+): /);- const errorName = nameMatch ? nameMatch[1] : '';- const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;- const firstLine = messageBody.split('\n')[0].trim();-- // Check for expect/assertion failure pattern.- const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);- if (matcherMatch) {- const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;- return {- elementName: 'failure',- type: matcherName,- message: firstLine,- };+ private _addFailureEntry(test: TestCase, entry: XMLEntry): 'failure' | 'error' {+ const errorInfo = classifyError(test);+ if (errorInfo) {+ entry.children!.push({+ name: errorInfo.elementName,+ attributes: { message: errorInfo.message, type: errorInfo.type },+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))+ });+ return errorInfo.elementName;}+ entry.children!.push({+ name: 'failure',+ attributes: {+ message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,+ type: 'FAILURE',+ },+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))+ });+ return 'failure';+ }++}- // Thrown error.+/**+ * Builds a Maven Surefire retry entry (<flakyFailure>/<flakyError> or <rerunFailure>/<rerunError>)+ * with per-result stackTrace, system-out, and system-err as children.+ */+function buildSurefireRetryEntry(result: TestResult, prefix: 'flaky' | 'rerun'): XMLEntry {+ const errorInfo = classifyResultError(result);+ const baseName = errorInfo?.elementName === 'error' ? 'Error' : 'Failure';+ const elementName = `${prefix}${baseName}`;+ const children: XMLEntry[] = [];+ const stackTrace = result.error?.stack || result.error?.message || result.error?.value || '';+ children.push({ name: 'stackTrace', text: stripAnsiEscapes(stackTrace) });+ const resultOut = result.stdout.map(s => s.toString()).join('');+ const resultErr = result.stderr.map(s => s.toString()).join('');+ if (resultOut)+ children.push({ name: 'system-out', text: resultOut });+ if (resultErr)+ children.push({ name: 'system-err', text: resultErr });+ return {+ name: elementName,+ attributes: { message: errorInfo?.message || '', type: errorInfo?.type || 'FAILURE', time: result.duration / 1000 },+ children,+ };+}++function classifyResultError(result: TestResult): { elementName: 'failure' | 'error'; type: string; message: string } | null {+ const error = result.error;+ if (!error)+ return null;++ const rawMessage = stripAnsiEscapes(error.message || error.value || '');++ // Parse "ErrorName: message" format from serialized error.+ const nameMatch = rawMessage.match(/^(\w+): /);+ const errorName = nameMatch ? nameMatch[1] : '';+ const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;+ const firstLine = messageBody.split('\n')[0].trim();++ // Check for expect/assertion failure pattern.+ const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);+ if (matcherMatch) {+ const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;return {- elementName: 'error',- type: errorName || 'Error',+ elementName: 'failure',+ type: matcherName,message: firstLine,};}++ // Thrown error.+ return {+ elementName: 'error',+ type: errorName || 'Error',+ message: firstLine,+ };+}++function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {+ for (const result of test.results) {+ const info = classifyResultError(result);+ if (info)+ return info;+ }return null;}diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.tsindex c30f7bca04..742c4b9ebf 100644--- a/packages/playwright/types/test.d.ts+++ b/packages/playwright/types/test.d.ts@@ -20,7 +20,7 @@ export * from 'playwright-core';export type BlobReporterOptions = { outputDir?: string, fileName?: string };export type ListReporterOptions = { printSteps?: boolean };-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };export type JsonReporterOptions = { outputFile?: string };export type HtmlReporterOptions = {outputFolder?: string;diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.tsindex 6781ef4049..ed8f1fe579 100644--- a/tests/playwright-test/reporter-junit.spec.ts+++ b/tests/playwright-test/reporter-junit.spec.ts@@ -633,5 +633,87 @@ for (const useIntermediateMergeReport of [false, true] as const) {expect(time).toBe(result.report.stats.duration / 1000);expect(time).toBeGreaterThan(1);});++ test('should emit flakyFailure for flaky tests when includeRetries is enabled', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ retries: 2,+ reporter: [['junit', { includeRetries: true }]]+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}, testInfo) => {+ expect(testInfo.retry).toBe(2);+ });+ `,+ }, { reporter: '' });+ const xml = parseXML(result.output);+ // Single testcase for the test; flaky tests count as passed.+ expect(xml['testsuites']['$']['tests']).toBe('1');+ expect(xml['testsuites']['$']['failures']).toBe('0');+ expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');+ const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];+ expect(testcase['$']['name']).toBe('one');+ // No <failure> element — test eventually passed.+ expect(testcase['failure']).toBeFalsy();+ // Two <flakyFailure> elements for the two failed attempts.+ expect(testcase['flakyFailure'].length).toBe(2);+ expect(testcase['flakyFailure'][0]['stackTrace']).toBeTruthy();+ expect(testcase['flakyFailure'][1]['stackTrace']).toBeTruthy();+ expect(result.exitCode).toBe(0);+ });++ test('should not include retries by default', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}, testInfo) => {+ expect(testInfo.retry).toBe(1);+ });+ `,+ }, { retries: 1, reporter: 'junit' });+ const xml = parseXML(result.output);+ // Default behavior: single testcase, no flakyFailure elements.+ expect(xml['testsuites']['$']['tests']).toBe('1');+ const testcases = xml['testsuites']['testsuite'][0]['testcase'];+ expect(testcases.length).toBe(1);+ expect(testcases[0]['$']['name']).toBe('one');+ expect(testcases[0]['flakyFailure']).toBeFalsy();+ expect(result.exitCode).toBe(0);+ });++ test('should emit rerunFailure for permanent failures when includeRetries is enabled', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'playwright.config.ts': `+ module.exports = {+ retries: 1,+ reporter: [['junit', { includeRetries: true }]]+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}) => {+ expect(1).toBe(0);+ });+ `,+ }, { reporter: '' });+ const xml = parseXML(result.output);+ // Single testcase; permanent failure counts as 1 test with 1 failure.+ expect(xml['testsuites']['$']['tests']).toBe('1');+ expect(xml['testsuites']['$']['failures']).toBe('1');+ expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('1');+ const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];+ expect(testcase['$']['name']).toBe('one');+ // <failure> for the first attempt.+ expect(testcase['failure']).toBeTruthy();+ // <rerunFailure> for the retry.+ expect(testcase['rerunFailure'].length).toBe(1);+ expect(testcase['rerunFailure'][0]['stackTrace']).toBeTruthy();+ expect(result.exitCode).toBe(1);+ });});}diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.tsindex e2529457cc..93bbfe5e2e 100644--- a/utils/generate_types/overrides-test.d.ts+++ b/utils/generate_types/overrides-test.d.ts@@ -19,7 +19,7 @@ export * from 'playwright-core';export type BlobReporterOptions = { outputDir?: string, fileName?: string };export type ListReporterOptions = { printSteps?: boolean };-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };export type JsonReporterOptions = { outputFile?: string };export type HtmlReporterOptions = {outputFolder?: string;