Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: replace private onMachineEnd with onReportConfigure/End (#39013)
Dmitry Gozman39bb72655203
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 27b48dc9a0..fc28d70f9b 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -25,7 +25,7 @@ export type JsonStackFrame = { file: string, line: number, column: number };export type JsonStdIOType = 'stdout' | 'stderr';-export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown'> & {+export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown' | 'shard'> & {// optional for backwards compatibilitytags?: reporterTypes.FullConfig['tags'],webServer?: reporterTypes.FullConfig['webServer'],@@ -445,11 +445,7 @@ export class TeleReporterReceiver {}private async _onEnd(result: JsonFullResult): Promise<void> {- await this._reporter.onEnd?.({- status: result.status,- startTime: new Date(result.startTime),- duration: result.duration,- });+ await this._reporter.onEnd?.(asFullResult(result));}private _onExit(): Promise<void> | void {@@ -457,7 +453,7 @@ export class TeleReporterReceiver {}private _parseConfig(config: JsonConfig): reporterTypes.FullConfig {- const result = { ...baseFullConfig, ...config };+ const result = asFullConfig(config);if (this._options.configOverrides) {result.configFile = this._options.configOverrides.configFile;result.reportSlowTests = this._options.configOverrides.reportSlowTests;@@ -846,3 +842,15 @@ export function computeTestCaseOutcome(test: reporterTypes.TestCase) {return 'unexpected'; // only failuresreturn 'flaky'; // expected+unexpected or skipped+unexpected}++export function asFullResult(result: JsonFullResult): reporterTypes.FullResult {+ return {+ status: result.status,+ startTime: new Date(result.startTime),+ duration: result.duration,+ };+}++export function asFullConfig(config: JsonConfig): reporterTypes.FullConfig {+ return { ...baseFullConfig, ...config };+}diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex c7997fe0ba..cd0fc9fd53 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -28,7 +28,7 @@ import { CommonReporterOptions, formatError, formatResultFailure, internalScreenimport { codeFrameColumns } from '../transform/babelBundle';import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';-import type { MachineEndResult, ReporterV2 } from './reporterV2';+import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test';import type * as api from '../../types/testReporter';import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';@@ -47,6 +47,12 @@ const isHtmlReportOption = (type: string): type is HtmlReportOpenOption => {return htmlReportOptions.includes(type as HtmlReportOpenOption);};+type MachineData = {+ config: api.FullConfig;+ result: api.FullResult;+ reportPath: string;+};+class HtmlReporter implements ReporterV2 {private config!: api.FullConfig;private suite!: api.Suite;@@ -58,7 +64,8 @@ class HtmlReporter implements ReporterV2 {private _host: string | undefined;private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;private _topLevelErrors: api.TestError[] = [];- private _machines: MachineEndResult[] = [];+ private _reportConfigs = new Map<string, api.FullConfig>();+ private _machines: MachineData[] = [];constructor(options: HtmlReporterConfigOptions & CommonReporterOptions) {this._options = options;@@ -122,8 +129,14 @@ class HtmlReporter implements ReporterV2 {this._topLevelErrors.push(error);}- onMachineEnd(result: MachineEndResult): void {- this._machines.push(result);+ onReportConfigure(params: ReportConfigureParams): void {+ this._reportConfigs.set(params.reportPath, params.config);+ }++ onReportEnd(params: ReportEndParams): void {+ const config = this._reportConfigs.get(params.reportPath);+ if (config)+ this._machines.push({ config, result: params.result, reportPath: params.reportPath });}async onEnd(result: api.FullResult) {@@ -255,7 +268,7 @@ class HtmlBuilder {this._attachmentsBaseURL = attachmentsBaseURL;}- async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineEndResult[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {+ async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineData[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {const data: DataMap = new Map();for (const projectSuite of projectSuites) {const projectName = projectSuite.project()!.name;@@ -303,11 +316,11 @@ class HtmlBuilder {stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },errors: topLevelErrors.map(error => formatError(internalScreen, error).message),options: this._options,- machines: machines.map(s => ({- duration: s.duration,- startTime: s.startTime.getTime(),- tag: s.tag,- shardIndex: s.shardIndex,+ machines: machines.map(machine => ({+ duration: machine.result.duration,+ startTime: machine.result.startTime.getTime(),+ tag: machine.config.tags,+ shardIndex: machine.config.shard?.current,})),};htmlReport.files.sort((f1, f2) => {diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex 8aed8f5280..13d662cc5a 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -22,7 +22,7 @@ import { ZipFile } from 'playwright-core/lib/utils';import { currentBlobReportVersion } from './blob';import { Multiplexer } from './multiplexer';import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool';-import { TeleReporterReceiver } from '../isomorphic/teleReceiver';+import { asFullConfig, asFullResult, TeleReporterReceiver } from '../isomorphic/teleReceiver';import { createReporters } from '../runner/reporters';import { relativeFilePath } from '../util';@@ -37,10 +37,10 @@ type StatusCallback = (message: string) => void;type ReportData = {eventPatchers: JsonEventPatchers;reportFile: string;+ zipFile: string;metadata: BlobReportMetadata;- tags: string[];- startTime: number;- duration: number;+ config: JsonConfig;+ fullResult: JsonFullResult;};export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {@@ -84,22 +84,24 @@ export async function createMergedReport(config: FullConfigInternal, dir: string};await dispatchEvents(eventData.prologue);- for (const { reportFile, eventPatchers, metadata, tags, startTime, duration } of eventData.reports) {+ for (const { reportFile, zipFile, eventPatchers, metadata, config, fullResult } of eventData.reports) {+ multiplexer.onReportConfigure({+ reportPath: zipFile,+ config: asFullConfig(config),+ });const reportJsonl = await fs.promises.readFile(reportFile);const events = parseTestEvents(reportJsonl);new JsonStringInternalizer(stringPool).traverse(events);eventPatchers.patchers.push(new AttachmentPathPatcher(dir));if (metadata.name)eventPatchers.patchers.push(new GlobalErrorPatcher(metadata.name));- if (tags.length)- eventPatchers.patchers.push(new GlobalErrorPatcher(tags.join(' ')));+ if (config?.tags?.length)+ eventPatchers.patchers.push(new GlobalErrorPatcher(config.tags.join(' ')));eventPatchers.patchEvents(events);await dispatchEvents(events);- multiplexer.onMachineEnd({- startTime: new Date(startTime),- duration,- tag: tags,- shardIndex: metadata.shard?.current,+ multiplexer.onReportEnd({+ reportPath: zipFile,+ result: asFullResult(fullResult),});}await dispatchEvents(eventData.epilogue);@@ -142,7 +144,7 @@ function splitBufferLines(buffer: Buffer) {}async function extractAndParseReports(dir: string, shardFiles: string[], internalizer: JsonStringInternalizer, printStatus: StatusCallback) {- const shardEvents: { file: string, localPath: string, metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[] = [];+ const shardEvents: { zipFile: string, reportFile: string, metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[] = [];await fs.promises.mkdir(path.join(dir, 'resources'), { recursive: true });const reportNames = new UniqueFileNameGenerator();@@ -152,10 +154,10 @@ async function extractAndParseReports(dir: string, shardFiles: string[], internaconst zipFile = new ZipFile(absolutePath);const entryNames = await zipFile.entries();for (const entryName of entryNames.sort()) {- let fileName = path.join(dir, entryName);+ let reportFile = path.join(dir, entryName);const content = await zipFile.read(entryName);if (entryName.endsWith('.jsonl')) {- fileName = reportNames.makeUnique(fileName);+ reportFile = reportNames.makeUnique(reportFile);let parsedEvents = parseCommonEvents(content);// Passing reviver to JSON.parse doesn't work, as the original strings// keep being used. To work around that we traverse the parsed events@@ -164,13 +166,13 @@ async function extractAndParseReports(dir: string, shardFiles: string[], internaconst metadata = findMetadata(parsedEvents, file);parsedEvents = modernizer.modernize(metadata.version, parsedEvents);shardEvents.push({- file,- localPath: fileName,+ zipFile: absolutePath,+ reportFile,metadata,parsedEvents});}- await fs.promises.writeFile(fileName, content);+ await fs.promises.writeFile(reportFile, content);}zipFile.close();}@@ -196,7 +198,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:const configureEvents: JsonOnConfigureEvent[] = [];const projectEvents: JsonOnProjectEvent[] = [];- const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[] = [];+ const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata }[] = [];const blobs = await extractAndParseReports(dir, shardReportFiles, internalizer, printStatus);// Sort by (report name; shard; file name), so that salt generation below is deterministic when:@@ -212,7 +214,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:const shardB = b.metadata.shard?.current ?? 0;if (shardA !== shardB)return shardA - shardB;- return a.file.localeCompare(b.file);+ return a.zipFile.localeCompare(b.zipFile);});printStatus(`merging events`);@@ -222,7 +224,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:for (let i = 0; i < blobs.length; ++i) {// Generate unique salt for each blob.- const { parsedEvents, metadata, localPath } = blobs[i];+ const { parsedEvents, metadata, reportFile, zipFile } = blobs[i];const eventPatchers = new JsonEventPatchers();eventPatchers.patchers.push(new IdsPatcher(stringPool,@@ -235,30 +237,28 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:eventPatchers.patchers.push(new PathSeparatorPatcher(metadata.pathSeparator));eventPatchers.patchEvents(parsedEvents);- let tags: string[] = [];- let startTime = 0;- let duration = 0;+ let config: JsonConfig | undefined;+ let fullResult: JsonFullResult | undefined;for (const event of parsedEvents) {if (event.method === 'onConfigure') {configureEvents.push(event);- tags = event.params.config.tags || [];+ config = event.params.config;} else if (event.method === 'onProject') {projectEvents.push(event);} else if (event.method === 'onEnd') {- endEvents.push({ event, metadata, tags });- startTime = event.params.result.startTime;- duration = event.params.result.duration;+ fullResult = event.params.result;+ endEvents.push({ event, metadata });}}// Save information about the reports to stream their test events later.reports.push({eventPatchers,- reportFile: localPath,+ reportFile,+ zipFile,metadata,- tags,- startTime,- duration,+ config: config!,+ fullResult: fullResult!,});}@@ -286,6 +286,7 @@ function mergeConfigureEvents(configureEvents: JsonOnConfigureEvent[], rootDirOvmaxFailures: 0,metadata: {},+ shard: null,rootDir: '',version: '',workers: 0,@@ -333,6 +334,7 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {...from.metadata,actualWorkers: (to.metadata.actualWorkers || 0) + (from.metadata.actualWorkers || 0),},+ shard: null,workers: to.workers + from.workers,};}diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.tsindex d4d2c3f4d5..97fcf27b79 100644--- a/packages/playwright/src/reporters/multiplexer.ts+++ b/packages/playwright/src/reporters/multiplexer.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import type { MachineEndResult, ReporterV2 } from './reporterV2';+import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';import type { Suite } from '../common/test';@@ -64,9 +64,14 @@ export class Multiplexer implements ReporterV2 {wrap(() => reporter.onTestEnd?.(test, result));}- onMachineEnd(result: MachineEndResult): void {+ onReportConfigure(params: ReportConfigureParams): void {for (const reporter of this._reporters)- wrap(() => reporter.onMachineEnd?.(result));+ wrap(() => reporter.onReportConfigure?.(params));+ }++ onReportEnd(params: ReportEndParams): void {+ for (const reporter of this._reporters)+ wrap(() => reporter.onReportEnd?.(params));}async onEnd(result: FullResult) {diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.tsindex 24507a059e..77405cd9ec 100644--- a/packages/playwright/src/reporters/reporterV2.ts+++ b/packages/playwright/src/reporters/reporterV2.ts@@ -16,11 +16,14 @@import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';-export interface MachineEndResult {- tag: string[];- shardIndex?: number;- startTime: Date;- duration: number;+export interface ReportConfigureParams {+ config: FullConfig;+ reportPath: string;+}++export interface ReportEndParams {+ reportPath: string;+ result: FullResult;}export interface ReporterV2 {@@ -31,7 +34,8 @@ export interface ReporterV2 {onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;onTestPaused?(test: TestCase, result: TestResult): Promise<void>;onTestEnd?(test: TestCase, result: TestResult): void;- onMachineEnd?(result: MachineEndResult): void;+ onReportConfigure?(params: ReportConfigureParams): void;+ onReportEnd?(params: ReportEndParams): void;onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;onExit?(): void | Promise<void>;onError?(error: TestError): void;diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex f4066ccfcf..7b70d15f34 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -186,6 +186,7 @@ export class TeleReporterEmitter implements ReporterV2 {maxFailures: config.maxFailures,metadata: config.metadata,rootDir: config.rootDir,+ shard: config.shard,version: config.version,workers: config.workers,globalSetup: config.globalSetup,diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.tsindex be99d83cf3..1ca22c1e4f 100644--- a/tests/playwright-test/reporter-blob.spec.ts+++ b/tests/playwright-test/reporter-blob.spec.ts@@ -2237,3 +2237,53 @@ test('should report duration across all shards', async ({ runInlineTest, mergeReconst { duration } = JSON.parse(outputLines[0]) as FullResult;expect(duration).toBeGreaterThanOrEqual(1500);});++test('shard chart', async ({ runInlineTest, writeFiles, showReport, page, mergeReports }) => {+ const reportDir = test.info().outputPath('blob-reports');+ await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ fullyParallel: true,+ tag: process.env.BOT_TAG,+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ import timers from 'timers/promises';+ test('one', async () => {+ await timers.setTimeout(100);+ });+ test('two', async () => {+ await timers.setTimeout(200);+ });+ test('three', async () => {+ await timers.setTimeout(300);+ });+ `,+ });++ await runInlineTest({}, { shard: '1/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });+ await runInlineTest({}, { shard: '2/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });+ await runInlineTest({}, { shard: '3/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });++ await runInlineTest({}, { shard: '1/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });+ await runInlineTest({}, { shard: '2/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });++ const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });+ expect(exitCode).toBe(0);+ await showReport();++ await page.getByRole('link', { name: 'Speedboard' }).click();++ await expect(page.getByRole('main')).toMatchAriaSnapshot(`+ - button "Timeline"+ - region:+ - img:+ - listitem /@linux/+ - listitem /@linux/+ - listitem /@linux/+ - listitem /@mac/+ - listitem /@mac/+ `);+});diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex 9f4d299887..262f927410 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -3306,54 +3306,6 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.getByRole('link', { name: 'previous' })).not.toBeVisible();});});--- test('shard chart', async ({ runInlineTest, writeFiles, showReport, page }) => {- test.skip(!useIntermediateMergeReport);-- await writeFiles({- 'playwright.config.ts': `- module.exports = {- fullyParallel: true,- tag: process.env.BOT_TAG,- };- `,- 'a.test.js': `- import { test, expect } from '@playwright/test';- import timers from 'timers/promises';- test('one', async () => {- await timers.setTimeout(100);- });- test('two', async () => {- await timers.setTimeout(200);- });- test('three', async () => {- await timers.setTimeout(300);- });- `,- });-- await runInlineTest({}, { reporter: 'dot,html', shard: '1/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });- await runInlineTest({}, { reporter: 'dot,html', shard: '2/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });- await runInlineTest({}, { reporter: 'dot,html', shard: '3/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });-- await runInlineTest({}, { reporter: 'dot,html', shard: '1/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });- await runInlineTest({}, { reporter: 'dot,html', shard: '2/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });-- await showReport();- await page.getByRole('link', { name: 'Speedboard' }).click();-- await expect(page.getByRole('main')).toMatchAriaSnapshot(`- - button "Timeline"- - region:- - img:- - listitem /@linux/- - listitem /@linux/- - listitem /@linux/- - listitem /@mac/- - listitem /@mac/- `);- });});}