Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(junit): add includeRetries option to emit each retry as separate testcase (#39464)

vazidmansuri0050154add954e0
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts
index 6b2c7561dc..80c1548474 100644
--- a/packages/playwright/src/reporters/junit.ts
+++ b/packages/playwright/src/reporters/junit.ts
@@ -24,7 +24,7 @@ import { stripAnsiEscapes } from '../util';
import type { ReporterV2 } from './reporterV2';
import type { JUnitReporterOptions } from '../../types/test';
-import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter';
+import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter';
class JUnitReporter implements ReporterV2 {
private config!: FullConfig;
@@ -38,10 +38,12 @@ class JUnitReporter implements ReporterV2 {
private resolvedOutputFile: string | undefined;
private stripANSIControlSequences = false;
private includeProjectInTestName = false;
+ private includeRetries = false;
constructor(options: JUnitReporterOptions & CommonReporterOptions) {
this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences);
this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName);
+ this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries);
this.configDir = options.configDir;
this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile;
}
@@ -143,6 +145,9 @@ class JUnitReporter implements ReporterV2 {
}
private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> {
+ const isRetried = this.includeRetries && test.results.length > 1;
+ const isFlaky = isRetried && test.ok();
+
const entry = {
name: 'testcase',
attributes: {
@@ -150,7 +155,14 @@ class JUnitReporter implements ReporterV2 {
name: namePrefix + test.titlePath().slice(3).join(' › '),
// filename
classname: suiteName,
- time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
+ // For flaky tests, use the last (successful) result's duration.
+ // For permanent failures with retries, use the first result's duration.
+ // Otherwise, use total duration across all results.
+ time: isFlaky
+ ? test.results[test.results.length - 1].duration / 1000
+ : isRetried
+ ? test.results[0].duration / 1000
+ : (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000
},
children: [] as XMLEntry[]
@@ -185,34 +197,40 @@ class JUnitReporter implements ReporterV2 {
}
let classification: 'failure' | 'error' | null = null;
- if (!test.ok()) {
- const errorInfo = classifyError(test);
- if (errorInfo) {
- classification = errorInfo.elementName;
- entry.children.push({
- name: errorInfo.elementName,
- attributes: {
- message: errorInfo.message,
- type: errorInfo.type,
- },
- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
- });
- } else {
- classification = 'failure';
- entry.children.push({
- name: 'failure',
- attributes: {
- message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
- type: 'FAILURE',
- },
- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
- });
+
+ if (isFlaky) {
+ // Flaky test (eventually passed): use Maven Surefire <flakyFailure>/<flakyError>.
+ // No <failure> element — flaky tests count as passed.
+ for (const result of test.results) {
+ if (result.status === 'passed' || result.status === 'skipped')
+ continue;
+ entry.children.push(buildSurefireRetryEntry(result, 'flaky'));
}
+ // classification stays null — flaky tests are not counted as failures.
+ } else if (isRetried) {
+ // Permanent failure (failed all retries): use <failure> + Maven Surefire <rerunFailure>/<rerunError>.
+ classification = this._addFailureEntry(test, entry);
+ // Add <rerunFailure>/<rerunError> for each subsequent retry.
+ for (let i = 1; i < test.results.length; i++) {
+ const result = test.results[i];
+ if (result.status === 'passed' || result.status === 'skipped')
+ continue;
+ entry.children.push(buildSurefireRetryEntry(result, 'rerun'));
+ }
+ } else if (!test.ok()) {
+ // Standard failure (no retries, or includeRetries is false).
+ classification = this._addFailureEntry(test, entry);
}
const systemOut: string[] = [];
const systemErr: string[] = [];
- for (const result of test.results) {
+ // When retries are included, top-level output comes from the primary result only:
+ // flaky → last (successful) result; permanent failure → first result.
+ // Without retries: all results (original behavior).
+ const outputResults = isRetried
+ ? [isFlaky ? test.results[test.results.length - 1] : test.results[0]]
+ : test.results;
+ for (const result of outputResults) {
for (const item of result.stdout)
systemOut.push(item.toString());
for (const item of result.stderr)
@@ -245,40 +263,92 @@ class JUnitReporter implements ReporterV2 {
entry.children.push({ name: 'system-err', text: systemErr.join('') });
return classification;
}
-}
-function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {
- for (const result of test.results) {
- const error = result.error;
- if (!error)
- continue;
-
- const rawMessage = stripAnsiEscapes(error.message || error.value || '');
-
- // Parse "ErrorName: message" format from serialized error.
- const nameMatch = rawMessage.match(/^(\w+): /);
- const errorName = nameMatch ? nameMatch[1] : '';
- const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;
- const firstLine = messageBody.split('\n')[0].trim();
-
- // Check for expect/assertion failure pattern.
- const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);
- if (matcherMatch) {
- const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;
- return {
- elementName: 'failure',
- type: matcherName,
- message: firstLine,
- };
+ private _addFailureEntry(test: TestCase, entry: XMLEntry): 'failure' | 'error' {
+ const errorInfo = classifyError(test);
+ if (errorInfo) {
+ entry.children!.push({
+ name: errorInfo.elementName,
+ attributes: { message: errorInfo.message, type: errorInfo.type },
+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+ });
+ return errorInfo.elementName;
}
+ entry.children!.push({
+ name: 'failure',
+ attributes: {
+ message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
+ type: 'FAILURE',
+ },
+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+ });
+ return 'failure';
+ }
+
+}
- // Thrown error.
+/**
+ * Builds a Maven Surefire retry entry (<flakyFailure>/<flakyError> or <rerunFailure>/<rerunError>)
+ * with per-result stackTrace, system-out, and system-err as children.
+ */
+function buildSurefireRetryEntry(result: TestResult, prefix: 'flaky' | 'rerun'): XMLEntry {
+ const errorInfo = classifyResultError(result);
+ const baseName = errorInfo?.elementName === 'error' ? 'Error' : 'Failure';
+ const elementName = `${prefix}${baseName}`;
+ const children: XMLEntry[] = [];
+ const stackTrace = result.error?.stack || result.error?.message || result.error?.value || '';
+ children.push({ name: 'stackTrace', text: stripAnsiEscapes(stackTrace) });
+ const resultOut = result.stdout.map(s => s.toString()).join('');
+ const resultErr = result.stderr.map(s => s.toString()).join('');
+ if (resultOut)
+ children.push({ name: 'system-out', text: resultOut });
+ if (resultErr)
+ children.push({ name: 'system-err', text: resultErr });
+ return {
+ name: elementName,
+ attributes: { message: errorInfo?.message || '', type: errorInfo?.type || 'FAILURE', time: result.duration / 1000 },
+ children,
+ };
+}
+
+function classifyResultError(result: TestResult): { elementName: 'failure' | 'error'; type: string; message: string } | null {
+ const error = result.error;
+ if (!error)
+ return null;
+
+ const rawMessage = stripAnsiEscapes(error.message || error.value || '');
+
+ // Parse "ErrorName: message" format from serialized error.
+ const nameMatch = rawMessage.match(/^(\w+): /);
+ const errorName = nameMatch ? nameMatch[1] : '';
+ const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;
+ const firstLine = messageBody.split('\n')[0].trim();
+
+ // Check for expect/assertion failure pattern.
+ const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);
+ if (matcherMatch) {
+ const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;
return {
- elementName: 'error',
- type: errorName || 'Error',
+ elementName: 'failure',
+ type: matcherName,
message: firstLine,
};
}
+
+ // Thrown error.
+ return {
+ elementName: 'error',
+ type: errorName || 'Error',
+ message: firstLine,
+ };
+}
+
+function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {
+ for (const result of test.results) {
+ const info = classifyResultError(result);
+ if (info)
+ return info;
+ }
return null;
}
diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts
index c30f7bca04..742c4b9ebf 100644
--- a/packages/playwright/types/test.d.ts
+++ b/packages/playwright/types/test.d.ts
@@ -20,7 +20,7 @@ export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
export type ListReporterOptions = { printSteps?: boolean };
-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };
+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
export type JsonReporterOptions = { outputFile?: string };
export type HtmlReporterOptions = {
outputFolder?: string;
diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts
index 6781ef4049..ed8f1fe579 100644
--- a/tests/playwright-test/reporter-junit.spec.ts
+++ b/tests/playwright-test/reporter-junit.spec.ts
@@ -633,5 +633,87 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(time).toBe(result.report.stats.duration / 1000);
expect(time).toBeGreaterThan(1);
});
+
+ test('should emit flakyFailure for flaky tests when includeRetries is enabled', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ retries: 2,
+ reporter: [['junit', { includeRetries: true }]]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('one', async ({}, testInfo) => {
+ expect(testInfo.retry).toBe(2);
+ });
+ `,
+ }, { reporter: '' });
+ const xml = parseXML(result.output);
+ // Single testcase for the test; flaky tests count as passed.
+ expect(xml['testsuites']['$']['tests']).toBe('1');
+ expect(xml['testsuites']['$']['failures']).toBe('0');
+ expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+ const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+ expect(testcase['$']['name']).toBe('one');
+ // No <failure> element — test eventually passed.
+ expect(testcase['failure']).toBeFalsy();
+ // Two <flakyFailure> elements for the two failed attempts.
+ expect(testcase['flakyFailure'].length).toBe(2);
+ expect(testcase['flakyFailure'][0]['stackTrace']).toBeTruthy();
+ expect(testcase['flakyFailure'][1]['stackTrace']).toBeTruthy();
+ expect(result.exitCode).toBe(0);
+ });
+
+ test('should not include retries by default', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('one', async ({}, testInfo) => {
+ expect(testInfo.retry).toBe(1);
+ });
+ `,
+ }, { retries: 1, reporter: 'junit' });
+ const xml = parseXML(result.output);
+ // Default behavior: single testcase, no flakyFailure elements.
+ expect(xml['testsuites']['$']['tests']).toBe('1');
+ const testcases = xml['testsuites']['testsuite'][0]['testcase'];
+ expect(testcases.length).toBe(1);
+ expect(testcases[0]['$']['name']).toBe('one');
+ expect(testcases[0]['flakyFailure']).toBeFalsy();
+ expect(result.exitCode).toBe(0);
+ });
+
+ test('should emit rerunFailure for permanent failures when includeRetries is enabled', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'playwright.config.ts': `
+ module.exports = {
+ retries: 1,
+ reporter: [['junit', { includeRetries: true }]]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('one', async ({}) => {
+ expect(1).toBe(0);
+ });
+ `,
+ }, { reporter: '' });
+ const xml = parseXML(result.output);
+ // Single testcase; permanent failure counts as 1 test with 1 failure.
+ expect(xml['testsuites']['$']['tests']).toBe('1');
+ expect(xml['testsuites']['$']['failures']).toBe('1');
+ expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('1');
+ const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+ expect(testcase['$']['name']).toBe('one');
+ // <failure> for the first attempt.
+ expect(testcase['failure']).toBeTruthy();
+ // <rerunFailure> for the retry.
+ expect(testcase['rerunFailure'].length).toBe(1);
+ expect(testcase['rerunFailure'][0]['stackTrace']).toBeTruthy();
+ expect(result.exitCode).toBe(1);
+ });
});
}
diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts
index e2529457cc..93bbfe5e2e 100644
--- a/utils/generate_types/overrides-test.d.ts
+++ b/utils/generate_types/overrides-test.d.ts
@@ -19,7 +19,7 @@ export * from 'playwright-core';
export type BlobReporterOptions = { outputDir?: string, fileName?: string };
export type ListReporterOptions = { printSteps?: boolean };
-export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean };
+export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean };
export type JsonReporterOptions = { outputFile?: string };
export type HtmlReporterOptions = {
outputFolder?: string;