Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(junit): use actual error message and distinguish errors from failures (#39350)
vazidmansuri0053bddc8c7921e
diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.tsindex 96f4288615..6b2c7561dc 100644--- a/packages/playwright/src/reporters/junit.ts+++ b/packages/playwright/src/reporters/junit.ts@@ -33,6 +33,7 @@ class JUnitReporter implements ReporterV2 {private timestamp!: Date;private totalTests = 0;private totalFailures = 0;+ private totalErrors = 0;private totalSkipped = 0;private resolvedOutputFile: string | undefined;private stripANSIControlSequences = false;@@ -79,7 +80,7 @@ class JUnitReporter implements ReporterV2 {tests: self.totalTests,failures: self.totalFailures,skipped: self.totalSkipped,- errors: 0,+ errors: self.totalErrors,time: result.duration / 1000},children@@ -100,6 +101,7 @@ class JUnitReporter implements ReporterV2 {let tests = 0;let skipped = 0;let failures = 0;+ let errors = 0;let duration = 0;const children: XMLEntry[] = [];const testCaseNamePrefix = projectName && this.includeProjectInTestName ? `[${projectName}] ` : '';@@ -108,16 +110,19 @@ class JUnitReporter implements ReporterV2 {++tests;if (test.outcome() === 'skipped')++skipped;- if (!test.ok())- ++failures;for (const result of test.results)duration += result.duration;- await this._addTestCase(suite.title, testCaseNamePrefix, test, children);+ const classification = await this._addTestCase(suite.title, testCaseNamePrefix, test, children);+ if (classification === 'error')+ ++errors;+ else if (classification === 'failure')+ ++failures;}this.totalTests += tests;this.totalSkipped += skipped;this.totalFailures += failures;+ this.totalErrors += errors;const entry: XMLEntry = {name: 'testsuite',@@ -129,7 +134,7 @@ class JUnitReporter implements ReporterV2 {failures,skipped,time: duration / 1000,- errors: 0,+ errors,},children};@@ -137,7 +142,7 @@ class JUnitReporter implements ReporterV2 {return entry;}- private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]) {+ private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> {const entry = {name: 'testcase',attributes: {@@ -176,18 +181,33 @@ class JUnitReporter implements ReporterV2 {if (test.outcome() === 'skipped') {entry.children.push({ name: 'skipped' });- return;+ return null;}+ let classification: 'failure' | 'error' | null = null;if (!test.ok()) {- entry.children.push({- name: 'failure',- attributes: {- message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,- type: 'FAILURE',- },- text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))- });+ const errorInfo = classifyError(test);+ if (errorInfo) {+ classification = errorInfo.elementName;+ entry.children.push({+ name: errorInfo.elementName,+ attributes: {+ message: errorInfo.message,+ type: errorInfo.type,+ },+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))+ });+ } else {+ classification = 'failure';+ entry.children.push({+ name: 'failure',+ attributes: {+ message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`,+ type: 'FAILURE',+ },+ text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test))+ });+ }}const systemOut: string[] = [];@@ -223,7 +243,43 @@ class JUnitReporter implements ReporterV2 {entry.children.push({ name: 'system-out', text: systemOut.join('') });if (systemErr.length)entry.children.push({ name: 'system-err', text: systemErr.join('') });+ return classification;+ }+}++function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null {+ for (const result of test.results) {+ const error = result.error;+ if (!error)+ continue;++ const rawMessage = stripAnsiEscapes(error.message || error.value || '');++ // Parse "ErrorName: message" format from serialized error.+ const nameMatch = rawMessage.match(/^(\w+): /);+ const errorName = nameMatch ? nameMatch[1] : '';+ const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage;+ const firstLine = messageBody.split('\n')[0].trim();++ // Check for expect/assertion failure pattern.+ const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/);+ if (matcherMatch) {+ const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`;+ return {+ elementName: 'failure',+ type: matcherName,+ message: firstLine,+ };+ }++ // Thrown error.+ return {+ elementName: 'error',+ type: errorName || 'Error',+ message: firstLine,+ };}+ return null;}type XMLEntry = {diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.tsindex 9947484903..6781ef4049 100644--- a/tests/playwright-test/reporter-junit.spec.ts+++ b/tests/playwright-test/reporter-junit.spec.ts@@ -62,14 +62,54 @@ for (const useIntermediateMergeReport of [false, true] as const) {const xml = parseXML(result.output);expect(xml['testsuites']['$']['tests']).toBe('1');expect(xml['testsuites']['$']['failures']).toBe('1');+ expect(xml['testsuites']['$']['errors']).toBe('0');const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];- expect(failure['$']['message']).toContain('a.test.js');- expect(failure['$']['message']).toContain('one');- expect(failure['$']['type']).toBe('FAILURE');+ expect(failure['$']['message']).toContain('expect(received).toBe(expected)');+ expect(failure['$']['type']).toBe('expect.toBe');expect(failure['_']).toContain('expect(1).toBe(0)');expect(result.exitCode).toBe(1);});+ test('should render thrown error as <error> element', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}) => {+ throw new Error('Boom');+ });+ `,+ }, { reporter: 'junit' });+ const xml = parseXML(result.output);+ expect(xml['testsuites']['$']['tests']).toBe('1');+ expect(xml['testsuites']['$']['failures']).toBe('0');+ expect(xml['testsuites']['$']['errors']).toBe('1');+ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');+ expect(xml['testsuites']['testsuite'][0]['$']['errors']).toBe('1');+ const error = xml['testsuites']['testsuite'][0]['testcase'][0]['error'][0];+ expect(error['$']['message']).toBe('Boom');+ expect(error['$']['type']).toBe('Error');+ expect(error['_']).toContain('Boom');+ expect(result.exitCode).toBe(1);+ });++ test('should render TypeError as <error> element with correct type', async ({ runInlineTest }) => {+ const result = await runInlineTest({+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ test('one', async ({}) => {+ const obj = null;+ obj.foo();+ });+ `,+ }, { reporter: 'junit' });+ const xml = parseXML(result.output);+ expect(xml['testsuites']['$']['errors']).toBe('1');+ expect(xml['testsuites']['$']['failures']).toBe('0');+ const error = xml['testsuites']['testsuite'][0]['testcase'][0]['error'][0];+ expect(error['$']['type']).toBe('TypeError');+ expect(result.exitCode).toBe(1);+ });+test('should render unexpected after retry', async ({ runInlineTest }) => {const result = await runInlineTest({'a.test.js': `