Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(junit): use actual error message and distinguish errors from failures (#39350)

vazidmansuri0053bddc8c7921e
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts
index 96f4288615..6b2c7561dc 100644
--- a/packages/playwright/src/reporters/junit.ts
+++ b/packages/playwright/src/reporters/junit.ts
@@ -33,6 +33,7 @@ class JUnitReporter implements ReporterV2 {
private timestamp!: Date;
private totalTests = 0;
private totalFailures = 0;
+ private totalErrors = 0;
private totalSkipped = 0;
private resolvedOutputFile: string | undefined;
private stripANSIControlSequences = false;
@@ -79,7 +80,7 @@ class JUnitReporter implements ReporterV2 {
tests: self.totalTests,
failures: self.totalFailures,
skipped: self.totalSkipped,
- errors: 0,
+ errors: self.totalErrors,
time: result.duration / 1000
},
children
@@ -100,6 +101,7 @@ class JUnitReporter implements ReporterV2 {
let tests = 0;
let skipped = 0;
let failures = 0;
+ let errors = 0;
let duration = 0;
const children: XMLEntry[] = [];
const testCaseNamePrefix = projectName && this.includeProjectInTestName ? `[${projectName}] ` : '';
@@ -108,16 +110,19 @@ class JUnitReporter implements ReporterV2 {
++tests;
if (test.outcome() === 'skipped')
++skipped;
- if (!test.ok())
- ++failures;
for (const result of test.results)
duration += result.duration;
- await this._addTestCase(suite.title, testCaseNamePrefix, test, children);
+ const classification = await this._addTestCase(suite.title, testCaseNamePrefix, test, children);
+ if (classification === 'error')
+ ++errors;
+ else if (classification === 'failure')
+ ++failures;
}
this.totalTests += tests;
this.totalSkipped += skipped;
this.totalFailures += failures;
+ this.totalErrors += errors;
const entry: XMLEntry = {
name: 'testsuite',
@@ -129,7 +134,7 @@ class JUnitReporter implements ReporterV2 {
failures,
skipped,
time: duration / 1000,
- errors: 0,
+ errors,
},
children
};
@@ -137,7 +142,7 @@ class JUnitReporter implements ReporterV2 {
return entry;
}
- private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]) {
+ private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> {
const entry = {
name: 'testcase',
attributes: {
@@ -176,18 +181,33 @@ class JUnitReporter implements ReporterV2 {
if (test.outcome() === 'skipped') {
entry.children.push({ name: 'skipped' });
- return;
+ return null;
}
+ let classification: 'failure' | 'error' | null = null;
if (!test.ok()) {
- entry.children.push({
- name: 'failure',
- attributes: {
- message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
- type: 'FAILURE',
- },
- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
- });
+ const errorInfo = classifyError(test);
+ if (errorInfo) {
+ classification = errorInfo.elementName;
+ entry.children.push({
+ name: errorInfo.elementName,
+ attributes: {
+ message: errorInfo.message,
+ type: errorInfo.type,
+ },
+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+ });
+ } else {
+ classification = 'failure';
+ entry.children.push({
+ name: 'failure',
+ attributes: {
+ message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,
+ type: 'FAILURE',
+ },
+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))
+ });
+ }
}
const systemOut: string[] = [];
@@ -223,7 +243,43 @@ class JUnitReporter implements ReporterV2 {
entry.children.push({ name: 'system-out', text: systemOut.join('') });
if (systemErr.length)
entry.children.push({ name: 'system-err', text: systemErr.join('') });
+ return classification;
+ }
+}
+
+function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {
+ for (const result of test.results) {
+ const error = result.error;
+ if (!error)
+ continue;
+
+ const rawMessage = stripAnsiEscapes(error.message || error.value || '');
+
+ // Parse "ErrorName: message" format from serialized error.
+ const nameMatch = rawMessage.match(/^(\w+): /);
+ const errorName = nameMatch ? nameMatch[1] : '';
+ const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;
+ const firstLine = messageBody.split('\n')[0].trim();
+
+ // Check for expect/assertion failure pattern.
+ const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);
+ if (matcherMatch) {
+ const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;
+ return {
+ elementName: 'failure',
+ type: matcherName,
+ message: firstLine,
+ };
+ }
+
+ // Thrown error.
+ return {
+ elementName: 'error',
+ type: errorName || 'Error',
+ message: firstLine,
+ };
}
+ return null;
}
type XMLEntry = {
diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts
index 9947484903..6781ef4049 100644
--- a/tests/playwright-test/reporter-junit.spec.ts
+++ b/tests/playwright-test/reporter-junit.spec.ts
@@ -62,14 +62,54 @@ for (const useIntermediateMergeReport of [false, true] as const) {
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['$']['failures']).toBe('1');
+ expect(xml['testsuites']['$']['errors']).toBe('0');
const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];
- expect(failure['$']['message']).toContain('a.test.js');
- expect(failure['$']['message']).toContain('one');
- expect(failure['$']['type']).toBe('FAILURE');
+ expect(failure['$']['message']).toContain('expect(received).toBe(expected)');
+ expect(failure['$']['type']).toBe('expect.toBe');
expect(failure['_']).toContain('expect(1).toBe(0)');
expect(result.exitCode).toBe(1);
});
+ test('should render thrown error as <error> element', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('one', async ({}) => {
+ throw new Error('Boom');
+ });
+ `,
+ }, { reporter: 'junit' });
+ const xml = parseXML(result.output);
+ expect(xml['testsuites']['$']['tests']).toBe('1');
+ expect(xml['testsuites']['$']['failures']).toBe('0');
+ expect(xml['testsuites']['$']['errors']).toBe('1');
+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
+ expect(xml['testsuites']['testsuite'][0]['$']['errors']).toBe('1');
+ const error = xml['testsuites']['testsuite'][0]['testcase'][0]['error'][0];
+ expect(error['$']['message']).toBe('Boom');
+ expect(error['$']['type']).toBe('Error');
+ expect(error['_']).toContain('Boom');
+ expect(result.exitCode).toBe(1);
+ });
+
+ test('should render TypeError as <error> element with correct type', async ({ runInlineTest }) => {
+ const result = await runInlineTest({
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ test('one', async ({}) => {
+ const obj = null;
+ obj.foo();
+ });
+ `,
+ }, { reporter: 'junit' });
+ const xml = parseXML(result.output);
+ expect(xml['testsuites']['$']['errors']).toBe('1');
+ expect(xml['testsuites']['$']['failures']).toBe('0');
+ const error = xml['testsuites']['testsuite'][0]['testcase'][0]['error'][0];
+ expect(error['$']['type']).toBe('TypeError');
+ expect(result.exitCode).toBe(1);
+ });
+
test('should render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `