Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: replace private onMachineEnd with onReportConfigure/End (#39013)

Dmitry Gozman39bb72655203
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index 27b48dc9a0..fc28d70f9b 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -25,7 +25,7 @@ export type JsonStackFrame = { file: string, line: number, column: number };
export type JsonStdIOType = 'stdout' | 'stderr';
-export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown'> & {
+export type JsonConfig = Pick<reporterTypes.FullConfig, 'configFile' | 'globalTimeout' | 'maxFailures' | 'metadata' | 'rootDir' | 'version' | 'workers' | 'globalSetup' | 'globalTeardown' | 'shard'> & {
// optional for backwards compatibility
tags?: reporterTypes.FullConfig['tags'],
webServer?: reporterTypes.FullConfig['webServer'],
@@ -445,11 +445,7 @@ export class TeleReporterReceiver {
}
private async _onEnd(result: JsonFullResult): Promise<void> {
- await this._reporter.onEnd?.({
- status: result.status,
- startTime: new Date(result.startTime),
- duration: result.duration,
- });
+ await this._reporter.onEnd?.(asFullResult(result));
}
private _onExit(): Promise<void> | void {
@@ -457,7 +453,7 @@ export class TeleReporterReceiver {
}
private _parseConfig(config: JsonConfig): reporterTypes.FullConfig {
- const result = { ...baseFullConfig, ...config };
+ const result = asFullConfig(config);
if (this._options.configOverrides) {
result.configFile = this._options.configOverrides.configFile;
result.reportSlowTests = this._options.configOverrides.reportSlowTests;
@@ -846,3 +842,15 @@ export function computeTestCaseOutcome(test: reporterTypes.TestCase) {
return 'unexpected'; // only failures
return 'flaky'; // expected+unexpected or skipped+unexpected
}
+
+export function asFullResult(result: JsonFullResult): reporterTypes.FullResult {
+ return {
+ status: result.status,
+ startTime: new Date(result.startTime),
+ duration: result.duration,
+ };
+}
+
+export function asFullConfig(config: JsonConfig): reporterTypes.FullConfig {
+ return { ...baseFullConfig, ...config };
+}
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index c7997fe0ba..cd0fc9fd53 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -28,7 +28,7 @@ import { CommonReporterOptions, formatError, formatResultFailure, internalScreen
import { codeFrameColumns } from '../transform/babelBundle';
import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';
-import type { MachineEndResult, ReporterV2 } from './reporterV2';
+import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';
import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test';
import type * as api from '../../types/testReporter';
import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';
@@ -47,6 +47,12 @@ const isHtmlReportOption = (type: string): type is HtmlReportOpenOption => {
return htmlReportOptions.includes(type as HtmlReportOpenOption);
};
+type MachineData = {
+ config: api.FullConfig;
+ result: api.FullResult;
+ reportPath: string;
+};
+
class HtmlReporter implements ReporterV2 {
private config!: api.FullConfig;
private suite!: api.Suite;
@@ -58,7 +64,8 @@ class HtmlReporter implements ReporterV2 {
private _host: string | undefined;
private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
private _topLevelErrors: api.TestError[] = [];
- private _machines: MachineEndResult[] = [];
+ private _reportConfigs = new Map<string, api.FullConfig>();
+ private _machines: MachineData[] = [];
constructor(options: HtmlReporterConfigOptions & CommonReporterOptions) {
this._options = options;
@@ -122,8 +129,14 @@ class HtmlReporter implements ReporterV2 {
this._topLevelErrors.push(error);
}
- onMachineEnd(result: MachineEndResult): void {
- this._machines.push(result);
+ onReportConfigure(params: ReportConfigureParams): void {
+ this._reportConfigs.set(params.reportPath, params.config);
+ }
+
+ onReportEnd(params: ReportEndParams): void {
+ const config = this._reportConfigs.get(params.reportPath);
+ if (config)
+ this._machines.push({ config, result: params.result, reportPath: params.reportPath });
}
async onEnd(result: api.FullResult) {
@@ -255,7 +268,7 @@ class HtmlBuilder {
this._attachmentsBaseURL = attachmentsBaseURL;
}
- async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineEndResult[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+ async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineData[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
const data: DataMap = new Map();
for (const projectSuite of projectSuites) {
const projectName = projectSuite.project()!.name;
@@ -303,11 +316,11 @@ class HtmlBuilder {
stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },
errors: topLevelErrors.map(error => formatError(internalScreen, error).message),
options: this._options,
- machines: machines.map(s => ({
- duration: s.duration,
- startTime: s.startTime.getTime(),
- tag: s.tag,
- shardIndex: s.shardIndex,
+ machines: machines.map(machine => ({
+ duration: machine.result.duration,
+ startTime: machine.result.startTime.getTime(),
+ tag: machine.config.tags,
+ shardIndex: machine.config.shard?.current,
})),
};
htmlReport.files.sort((f1, f2) => {
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index 8aed8f5280..13d662cc5a 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -22,7 +22,7 @@ import { ZipFile } from 'playwright-core/lib/utils';
import { currentBlobReportVersion } from './blob';
import { Multiplexer } from './multiplexer';
import { JsonStringInternalizer, StringInternPool } from '../isomorphic/stringInternPool';
-import { TeleReporterReceiver } from '../isomorphic/teleReceiver';
+import { asFullConfig, asFullResult, TeleReporterReceiver } from '../isomorphic/teleReceiver';
import { createReporters } from '../runner/reporters';
import { relativeFilePath } from '../util';
@@ -37,10 +37,10 @@ type StatusCallback = (message: string) => void;
type ReportData = {
eventPatchers: JsonEventPatchers;
reportFile: string;
+ zipFile: string;
metadata: BlobReportMetadata;
- tags: string[];
- startTime: number;
- duration: number;
+ config: JsonConfig;
+ fullResult: JsonFullResult;
};
export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {
@@ -84,22 +84,24 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
};
await dispatchEvents(eventData.prologue);
- for (const { reportFile, eventPatchers, metadata, tags, startTime, duration } of eventData.reports) {
+ for (const { reportFile, zipFile, eventPatchers, metadata, config, fullResult } of eventData.reports) {
+ multiplexer.onReportConfigure({
+ reportPath: zipFile,
+ config: asFullConfig(config),
+ });
const reportJsonl = await fs.promises.readFile(reportFile);
const events = parseTestEvents(reportJsonl);
new JsonStringInternalizer(stringPool).traverse(events);
eventPatchers.patchers.push(new AttachmentPathPatcher(dir));
if (metadata.name)
eventPatchers.patchers.push(new GlobalErrorPatcher(metadata.name));
- if (tags.length)
- eventPatchers.patchers.push(new GlobalErrorPatcher(tags.join(' ')));
+ if (config?.tags?.length)
+ eventPatchers.patchers.push(new GlobalErrorPatcher(config.tags.join(' ')));
eventPatchers.patchEvents(events);
await dispatchEvents(events);
- multiplexer.onMachineEnd({
- startTime: new Date(startTime),
- duration,
- tag: tags,
- shardIndex: metadata.shard?.current,
+ multiplexer.onReportEnd({
+ reportPath: zipFile,
+ result: asFullResult(fullResult),
});
}
await dispatchEvents(eventData.epilogue);
@@ -142,7 +144,7 @@ function splitBufferLines(buffer: Buffer) {
}
async function extractAndParseReports(dir: string, shardFiles: string[], internalizer: JsonStringInternalizer, printStatus: StatusCallback) {
- const shardEvents: { file: string, localPath: string, metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[] = [];
+ const shardEvents: { zipFile: string, reportFile: string, metadata: BlobReportMetadata, parsedEvents: JsonEvent[] }[] = [];
await fs.promises.mkdir(path.join(dir, 'resources'), { recursive: true });
const reportNames = new UniqueFileNameGenerator();
@@ -152,10 +154,10 @@ async function extractAndParseReports(dir: string, shardFiles: string[], interna
const zipFile = new ZipFile(absolutePath);
const entryNames = await zipFile.entries();
for (const entryName of entryNames.sort()) {
- let fileName = path.join(dir, entryName);
+ let reportFile = path.join(dir, entryName);
const content = await zipFile.read(entryName);
if (entryName.endsWith('.jsonl')) {
- fileName = reportNames.makeUnique(fileName);
+ reportFile = reportNames.makeUnique(reportFile);
let parsedEvents = parseCommonEvents(content);
// Passing reviver to JSON.parse doesn't work, as the original strings
// keep being used. To work around that we traverse the parsed events
@@ -164,13 +166,13 @@ async function extractAndParseReports(dir: string, shardFiles: string[], interna
const metadata = findMetadata(parsedEvents, file);
parsedEvents = modernizer.modernize(metadata.version, parsedEvents);
shardEvents.push({
- file,
- localPath: fileName,
+ zipFile: absolutePath,
+ reportFile,
metadata,
parsedEvents
});
}
- await fs.promises.writeFile(fileName, content);
+ await fs.promises.writeFile(reportFile, content);
}
zipFile.close();
}
@@ -196,7 +198,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
const configureEvents: JsonOnConfigureEvent[] = [];
const projectEvents: JsonOnProjectEvent[] = [];
- const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[] = [];
+ const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata }[] = [];
const blobs = await extractAndParseReports(dir, shardReportFiles, internalizer, printStatus);
// Sort by (report name; shard; file name), so that salt generation below is deterministic when:
@@ -212,7 +214,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
const shardB = b.metadata.shard?.current ?? 0;
if (shardA !== shardB)
return shardA - shardB;
- return a.file.localeCompare(b.file);
+ return a.zipFile.localeCompare(b.zipFile);
});
printStatus(`merging events`);
@@ -222,7 +224,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
for (let i = 0; i < blobs.length; ++i) {
// Generate unique salt for each blob.
- const { parsedEvents, metadata, localPath } = blobs[i];
+ const { parsedEvents, metadata, reportFile, zipFile } = blobs[i];
const eventPatchers = new JsonEventPatchers();
eventPatchers.patchers.push(new IdsPatcher(
stringPool,
@@ -235,30 +237,28 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
eventPatchers.patchers.push(new PathSeparatorPatcher(metadata.pathSeparator));
eventPatchers.patchEvents(parsedEvents);
- let tags: string[] = [];
- let startTime = 0;
- let duration = 0;
+ let config: JsonConfig | undefined;
+ let fullResult: JsonFullResult | undefined;
for (const event of parsedEvents) {
if (event.method === 'onConfigure') {
configureEvents.push(event);
- tags = event.params.config.tags || [];
+ config = event.params.config;
} else if (event.method === 'onProject') {
projectEvents.push(event);
} else if (event.method === 'onEnd') {
- endEvents.push({ event, metadata, tags });
- startTime = event.params.result.startTime;
- duration = event.params.result.duration;
+ fullResult = event.params.result;
+ endEvents.push({ event, metadata });
}
}
// Save information about the reports to stream their test events later.
reports.push({
eventPatchers,
- reportFile: localPath,
+ reportFile,
+ zipFile,
metadata,
- tags,
- startTime,
- duration,
+ config: config!,
+ fullResult: fullResult!,
});
}
@@ -286,6 +286,7 @@ function mergeConfigureEvents(configureEvents: JsonOnConfigureEvent[], rootDirOv
maxFailures: 0,
metadata: {
},
+ shard: null,
rootDir: '',
version: '',
workers: 0,
@@ -333,6 +334,7 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {
...from.metadata,
actualWorkers: (to.metadata.actualWorkers || 0) + (from.metadata.actualWorkers || 0),
},
+ shard: null,
workers: to.workers + from.workers,
};
}
diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts
index d4d2c3f4d5..97fcf27b79 100644
--- a/packages/playwright/src/reporters/multiplexer.ts
+++ b/packages/playwright/src/reporters/multiplexer.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import type { MachineEndResult, ReporterV2 } from './reporterV2';
+import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2';
import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
import type { Suite } from '../common/test';
@@ -64,9 +64,14 @@ export class Multiplexer implements ReporterV2 {
wrap(() => reporter.onTestEnd?.(test, result));
}
- onMachineEnd(result: MachineEndResult): void {
+ onReportConfigure(params: ReportConfigureParams): void {
for (const reporter of this._reporters)
- wrap(() => reporter.onMachineEnd?.(result));
+ wrap(() => reporter.onReportConfigure?.(params));
+ }
+
+ onReportEnd(params: ReportEndParams): void {
+ for (const reporter of this._reporters)
+ wrap(() => reporter.onReportEnd?.(params));
}
async onEnd(result: FullResult) {
diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts
index 24507a059e..77405cd9ec 100644
--- a/packages/playwright/src/reporters/reporterV2.ts
+++ b/packages/playwright/src/reporters/reporterV2.ts
@@ -16,11 +16,14 @@
import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
-export interface MachineEndResult {
- tag: string[];
- shardIndex?: number;
- startTime: Date;
- duration: number;
+export interface ReportConfigureParams {
+ config: FullConfig;
+ reportPath: string;
+}
+
+export interface ReportEndParams {
+ reportPath: string;
+ result: FullResult;
}
export interface ReporterV2 {
@@ -31,7 +34,8 @@ export interface ReporterV2 {
onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
onTestPaused?(test: TestCase, result: TestResult): Promise<void>;
onTestEnd?(test: TestCase, result: TestResult): void;
- onMachineEnd?(result: MachineEndResult): void;
+ onReportConfigure?(params: ReportConfigureParams): void;
+ onReportEnd?(params: ReportEndParams): void;
onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
onExit?(): void | Promise<void>;
onError?(error: TestError): void;
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index f4066ccfcf..7b70d15f34 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -186,6 +186,7 @@ export class TeleReporterEmitter implements ReporterV2 {
maxFailures: config.maxFailures,
metadata: config.metadata,
rootDir: config.rootDir,
+ shard: config.shard,
version: config.version,
workers: config.workers,
globalSetup: config.globalSetup,
diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts
index be99d83cf3..1ca22c1e4f 100644
--- a/tests/playwright-test/reporter-blob.spec.ts
+++ b/tests/playwright-test/reporter-blob.spec.ts
@@ -2237,3 +2237,53 @@ test('should report duration across all shards', async ({ runInlineTest, mergeRe
const { duration } = JSON.parse(outputLines[0]) as FullResult;
expect(duration).toBeGreaterThanOrEqual(1500);
});
+
+test('shard chart', async ({ runInlineTest, writeFiles, showReport, page, mergeReports }) => {
+ const reportDir = test.info().outputPath('blob-reports');
+ await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = {
+ fullyParallel: true,
+ tag: process.env.BOT_TAG,
+ reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+ };
+ `,
+ 'a.test.js': `
+ import { test, expect } from '@playwright/test';
+ import timers from 'timers/promises';
+ test('one', async () => {
+ await timers.setTimeout(100);
+ });
+ test('two', async () => {
+ await timers.setTimeout(200);
+ });
+ test('three', async () => {
+ await timers.setTimeout(300);
+ });
+ `,
+ });
+
+ await runInlineTest({}, { shard: '1/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
+ await runInlineTest({}, { shard: '2/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
+ await runInlineTest({}, { shard: '3/3' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
+
+ await runInlineTest({}, { shard: '1/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
+ await runInlineTest({}, { shard: '2/2' }, { PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
+
+ const { exitCode } = await mergeReports(reportDir, { 'PLAYWRIGHT_HTML_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
+ expect(exitCode).toBe(0);
+ await showReport();
+
+ await page.getByRole('link', { name: 'Speedboard' }).click();
+
+ await expect(page.getByRole('main')).toMatchAriaSnapshot(`
+ - button "Timeline"
+ - region:
+ - img:
+ - listitem /@linux/
+ - listitem /@linux/
+ - listitem /@linux/
+ - listitem /@mac/
+ - listitem /@mac/
+ `);
+});
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index 9f4d299887..262f927410 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -3306,54 +3306,6 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await expect(page.getByRole('link', { name: 'previous' })).not.toBeVisible();
});
});
-
-
- test('shard chart', async ({ runInlineTest, writeFiles, showReport, page }) => {
- test.skip(!useIntermediateMergeReport);
-
- await writeFiles({
- 'playwright.config.ts': `
- module.exports = {
- fullyParallel: true,
- tag: process.env.BOT_TAG,
- };
- `,
- 'a.test.js': `
- import { test, expect } from '@playwright/test';
- import timers from 'timers/promises';
- test('one', async () => {
- await timers.setTimeout(100);
- });
- test('two', async () => {
- await timers.setTimeout(200);
- });
- test('three', async () => {
- await timers.setTimeout(300);
- });
- `,
- });
-
- await runInlineTest({}, { reporter: 'dot,html', shard: '1/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
- await runInlineTest({}, { reporter: 'dot,html', shard: '2/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
- await runInlineTest({}, { reporter: 'dot,html', shard: '3/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });
-
- await runInlineTest({}, { reporter: 'dot,html', shard: '1/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
- await runInlineTest({}, { reporter: 'dot,html', shard: '2/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });
-
- await showReport();
- await page.getByRole('link', { name: 'Speedboard' }).click();
-
- await expect(page.getByRole('main')).toMatchAriaSnapshot(`
- - button "Timeline"
- - region:
- - img:
- - listitem /@linux/
- - listitem /@linux/
- - listitem /@linux/
- - listitem /@mac/
- - listitem /@mac/
- `);
- });
});
}