Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(reporter): report per-shard durations (#38623)
Simon Knott5579232eacc3
diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.mdindex cd6cf1e643..0d1300b4ad 100644--- a/docs/src/test-reporter-api/class-reporter.md+++ b/docs/src/test-reporter-api/class-reporter.md@@ -132,6 +132,11 @@ Reporter is allowed to override the status and hence affect the exit code of the- `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Test run status.- `startTime` <[Date]> Test run start wall time.- `duration` <[int]> Test run duration in milliseconds.+ - `shards` <[Array]<[Object]>> Only present on merged reports+ - `shardIndex` ?<[int]> The index of the shard, one-based.+ - `tag` ?<[Array]<[string]>> Global [`property: TestConfig.tag`] that differentiates CI environments+ - `startTime` <[Date]> Start wall time of shard.+ - `duration` <[int]> Shard run duration in milliseconds.Result of the full test run, `status` can be one of:* `'passed'` - Everything went as expected.diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.mdindex 278a2dbaf6..eb5f4d7262 100644--- a/docs/src/test-reporter-api/class-testresult.md+++ b/docs/src/test-reporter-api/class-testresult.md@@ -103,6 +103,6 @@ The index of the worker between `0` and `workers - 1`. It is guaranteed that wor## property: TestResult.shardIndex* since: v1.58-- type: <[int]>+- type: ?<[int]>-The index of the shard between `0` and `shards - 1`.+The index of the shard between `1` and [`shards`](../test-sharding.md).diff --git a/packages/html-reporter/src/barchart.css b/packages/html-reporter/src/barchart.cssnew file mode 100644index 0000000000..08478189a1--- /dev/null+++ b/packages/html-reporter/src/barchart.css@@ -0,0 +1,28 @@+/*+ Copyright (c) Microsoft Corporation.++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+*/++.barchart-bar {+ transition: opacity 0.2s;+ cursor: pointer;+ outline: none;+}++.barchart-bar:hover,+.barchart-bar:focus {+ opacity: 0.8;+ stroke: var(--color-fg-default);+ stroke-width: 2;+}diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsxnew file mode 100644index 0000000000..a4b52d501c--- /dev/null+++ b/packages/html-reporter/src/barchart.tsx@@ -0,0 +1,234 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import './barchart.css';++const formatDuration = (ms: number): string => {+ const totalSeconds = Math.round(ms / 1000);+ const minutes = Math.floor(totalSeconds / 60);+ const seconds = totalSeconds % 60;+ if (minutes === 0)+ return `${seconds}s`;+ return `${minutes}m ${seconds}s`;+};++export const GroupedBarChart = ({+ data,+ groups,+ series,+}: {+ data: number[][];+ groups: string[];+ series: string[];+}) => {+ const width = 800;++ // Calculate left margin based on longest group name+ // Rough estimate: 7 pixels per character at fontSize 12+ const maxGroupNameLength = Math.max(...groups.map(g => g.length));+ const estimatedTextWidth = maxGroupNameLength * 7;+ const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));++ const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };+ const chartWidth = width - margin.left - margin.right;++ const maxValue = Math.max(...data.flat());++ let tickInterval: number;+ let formatTickLabel: (i: number) => string;++ if (maxValue < 60 * 1000) {+ tickInterval = 10 * 1000;+ formatTickLabel = i => `${i * 10}s`;+ } else if (maxValue < 5 * 60 * 1000) {+ tickInterval = 30 * 1000;+ formatTickLabel = i => {+ const seconds = i * 30;+ const mins = Math.floor(seconds / 60);+ const secs = seconds % 60;+ return secs === 0 ? `${mins}m` : `${mins}m${secs}s`;+ };+ } else if (maxValue < 30 * 60 * 1000) {+ tickInterval = 5 * 60 * 1000;+ formatTickLabel = i => `${i * 5}m`;+ } else {+ tickInterval = 10 * 60 * 1000;+ formatTickLabel = i => `${i * 10}m`;+ }++ const maxRounded = Math.ceil(maxValue / tickInterval) * tickInterval;+ const xScale = chartWidth / maxRounded;++ // Calculate the number of actual bars per group (non-zero values)+ const barsPerGroup = data.map(group => group.length);++ // Allocate space proportionally based on number of bars+ const barHeight = 20; // Fixed bar height+ const barSpacing = 4;+ const groupPadding = 12;++ // Calculate Y positions for each group based on their bar count+ const groupYPositions: number[] = [];+ let currentY = 0;+ for (let i = 0; i < groups.length; i++) {+ groupYPositions.push(currentY);+ const groupHeight = barsPerGroup[i] * barHeight + (barsPerGroup[i] - 1) * barSpacing + groupPadding;+ currentY += groupHeight;+ }++ const contentHeight = currentY;++ const xTicks = [];+ const numberOfTicks = Math.ceil(maxRounded / tickInterval);+ for (let i = 0; i <= numberOfTicks; i++) {+ xTicks.push({+ x: i * tickInterval * xScale,+ label: formatTickLabel(i)+ });+ }++ const height = contentHeight + margin.top + margin.bottom;++ return (+ <svg+ viewBox={`0 0 ${width} ${height}`}+ preserveAspectRatio='xMidYMid meet'+ style={{ width: '100%', height: 'auto' }}+ role='img'+ >+ <g transform={`translate(${margin.left}, ${margin.top})`} role='presentation'>+ {xTicks.map(({ x, label }, i) => (+ <g key={i} aria-hidden='true'>+ <line+ x1={x}+ y1={0}+ x2={x}+ y2={contentHeight}+ stroke='var(--color-border-muted)'+ strokeWidth='1'+ />+ <text+ x={x}+ y={contentHeight + 20}+ textAnchor='middle'+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ >+ {label}+ </text>+ </g>+ ))}++ {groups.map((group, groupIndex) => {+ const groupY = groupYPositions[groupIndex];+ let barIndex = 0;++ return (+ <g key={groupIndex} role='list' aria-label={group}>+ {series.map((seriesName, seriesIndex) => {+ const value = data[groupIndex][seriesIndex];+ if (value === undefined || Number.isNaN(value))+ return null;++ const barWidth = value * xScale;+ const x = 0;+ const y = groupY + barIndex * (barHeight + barSpacing);+ barIndex++;++ const colors = ['var(--color-scale-yellow-3)', 'var(--color-scale-orange-4)', 'var(--color-scale-blue-3)', 'var(--color-scale-green-3)'];+ const color = colors[seriesIndex % colors.length];++ return (+ <g+ key={`${groupIndex}-${seriesIndex}`}+ role='listitem'+ aria-label={`${seriesName}: ${formatDuration(value)}`}+ >+ <rect+ className='barchart-bar'+ x={x}+ y={y}+ width={barWidth}+ height={barHeight}+ fill={color}+ rx='2'+ tabIndex={0}+ >+ <title>{`${seriesName}: ${formatDuration(value)}`}</title>+ </rect>+ <text+ x={barWidth + 6}+ y={y + barHeight / 2}+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ aria-hidden='true'+ >+ {formatDuration(value)}+ </text>+ </g>+ );+ })}+ </g>+ );+ })}++ {groups.map((group, groupIndex) => {+ const groupY = groupYPositions[groupIndex];+ const actualBars = barsPerGroup[groupIndex];+ const groupHeight = actualBars * barHeight + (actualBars - 1) * barSpacing;+ const labelY = groupY + groupHeight / 2;++ return (+ <text+ key={groupIndex}+ x={-10}+ y={labelY}+ textAnchor='end'+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ aria-hidden='true'+ >+ {group}+ </text>+ );+ })}++ <line+ x1={0}+ y1={0}+ x2={0}+ y2={contentHeight}+ stroke='var(--color-fg-muted)'+ strokeWidth='1'+ aria-hidden='true'+ />++ <line+ x1={0}+ y1={contentHeight}+ x2={chartWidth}+ y2={contentHeight}+ stroke='var(--color-fg-muted)'+ strokeWidth='1'+ aria-hidden='true'+ />+ </g>+ </svg>+ );+};diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsxindex b0beb0267a..b461a2272e 100644--- a/packages/html-reporter/src/speedboard.tsx+++ b/packages/html-reporter/src/speedboard.tsx@@ -19,9 +19,12 @@ import { LoadedReport } from './loadedReport';import { TestFileView } from './testFileView';import * as icons from './icons';import { TestCaseSummary } from './types';+import { AutoChip } from './chip';+import { GroupedBarChart } from './barchart';export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {return <>+ <Shards report={report} /><SlowestTests report={report} tests={tests} /></>;}@@ -46,3 +49,37 @@ export function SlowestTests({ report, tests }: { report: LoadedReport, tests: T}/>;}++export function Shards({ report }: { report: LoadedReport }) {+ const shards = report.json().shards;+ if (shards.length === 0)+ return null;++ let clash = false;+ const bots: Record<string, number[]> = {};+ for (const shard of shards) {+ const botName = shard.tag.join(' ');+ bots[botName] ??= [];+ const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0);+ if (bots[botName][shardIndex] !== undefined)+ clash = true;+ bots[botName][shardIndex] = shard.duration;+ }++ const maxSeries = Math.max(...Object.values(bots).map(shardDurations => shardDurations.length));++ return <AutoChip header='Shard Duration'>+ <GroupedBarChart+ data={Object.values(bots)}+ groups={Object.keys(bots)}+ series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}+ />+ {clash && <div style={{ marginTop: 8 }}>+ <icons.warning />+ Some shards could not be differentiated because of missing global tags.+ Please refer to <a href='https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-environments' target='_blank' rel='noopener noreferrer'>+ the docs+ </a> on how to fix this.+ </div>}+ </AutoChip>;+}diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.tsindex 270a8cb983..ce913a55a9 100644--- a/packages/html-reporter/src/types.d.ts+++ b/packages/html-reporter/src/types.d.ts@@ -49,6 +49,12 @@ export type HTMLReport = {projectNames: string[];startTime: number;duration: number;+ shards: {+ shardIndex?: number;+ tag: string[];+ startTime: number;+ duration: number;+ }[];errors: string[]; // Top-level errors that are not attributed to any test.options: HTMLReportOptions;};diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex 10c967d33c..c113c8b9b6 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -87,7 +87,7 @@ export type JsonTestResultStart = {retry: number;workerIndex: number;parallelIndex: number;- shardIndex: number;+ shardIndex?: number;startTime: number;};@@ -130,6 +130,12 @@ export type JsonFullResult = {status: reporterTypes.FullResult['status'];startTime: number;duration: number;+ shards?: {+ shardIndex?: number;+ tag: string[];+ startTime: number;+ duration: number;+ }[];};export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent@@ -450,6 +456,12 @@ export class TeleReporterReceiver {status: result.status,startTime: new Date(result.startTime),duration: result.duration,+ shards: result.shards?.map(s => ({+ shardIndex: s.shardIndex,+ tag: s.tag,+ startTime: new Date(s.startTime),+ duration: s.duration,+ })) ?? [],});}@@ -723,7 +735,7 @@ export class TeleTestResult implements reporterTypes.TestResult {retry: reporterTypes.TestResult['retry'];parallelIndex: reporterTypes.TestResult['parallelIndex'] = -1;workerIndex: reporterTypes.TestResult['workerIndex'] = -1;- shardIndex: reporterTypes.TestResult['shardIndex'] = -1;+ shardIndex: reporterTypes.TestResult['shardIndex'];duration: reporterTypes.TestResult['duration'] = -1;stdout: reporterTypes.TestResult['stdout'] = [];stderr: reporterTypes.TestResult['stderr'] = [];diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex b97e571dac..ad1942a88a 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -298,6 +298,12 @@ class HtmlBuilder {stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },errors: topLevelErrors.map(error => formatError(internalScreen, error).message),options: this._options,+ shards: result.shards?.map(s => ({+ shardIndex: s.shardIndex,+ tag: s.tag,+ startTime: s.startTime.getTime(),+ duration: s.duration,+ })) ?? []};htmlReport.files.sort((f1, f2) => {const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.tsindex 7b0ac95b82..e68d34ff25 100644--- a/packages/playwright/src/reporters/internalReporter.ts+++ b/packages/playwright/src/reporters/internalReporter.ts@@ -86,6 +86,7 @@ export class InternalReporter implements ReporterV2 {...result,startTime: this._startTime!,duration: monotonicTime() - this._monotonicStartTime!,+ shards: [],});}diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex 81d46321c3..55624b0714 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -188,7 +188,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:const configureEvents: JsonOnConfigureEvent[] = [];const projectEvents: JsonOnProjectEvent[] = [];- const endEvents: JsonOnEndEvent[] = [];+ const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[] = [];const blobs = await extractAndParseReports(dir, shardReportFiles, internalizer, printStatus);// Sort by (report name; shard; file name), so that salt generation below is deterministic when:@@ -235,7 +235,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:} else if (event.method === 'onProject') {projectEvents.push(event);} else if (event.method === 'onEnd') {- endEvents.push(event);+ endEvents.push({ event, metadata, tags });}}@@ -323,12 +323,13 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {};}-function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent {+function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[]): JsonEvent {let startTime = endEvents.length ? 10000000000000 : Date.now();let status: JsonFullResult['status'] = 'passed';let endTime: number = 0;+ const shards: JsonFullResult['shards'] = [];- for (const event of endEvents) {+ for (const { event, metadata, tags } of endEvents) {const shardResult = event.params.result;if (shardResult.status === 'failed')status = 'failed';@@ -338,11 +339,19 @@ function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent {status = 'interrupted';startTime = Math.min(startTime, shardResult.startTime);endTime = Math.max(endTime, shardResult.startTime + shardResult.duration);++ shards.push({+ shardIndex: metadata.shard?.current,+ tag: tags,+ startTime: shardResult.startTime,+ duration: shardResult.duration,+ });}const result: JsonFullResult = {status,startTime,duration: endTime - startTime,+ shards,};return {method: 'onEnd',diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex 2b65ed5ba9..0648706ecb 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -166,6 +166,12 @@ export class TeleReporterEmitter implements ReporterV2 {status: result.status,startTime: result.startTime.getTime(),duration: result.duration,+ shards: result.shards?.map(s => ({+ shardIndex: s.shardIndex,+ tag: s.tag,+ startTime: +s.startTime,+ duration: s.duration,+ })),};this._messageSink({method: 'onEnd',diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 5c9f2ae834..dd8defedda 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -320,7 +320,7 @@ class JobDispatcher {this._dataByTestId.set(test.id, { test, result, steps: new Map() });result.parallelIndex = this._parallelIndex;result.workerIndex = this._workerIndex;- result.shardIndex = this._config.config.shard?.current ?? 0;+ result.shardIndex = this._config.config.shard?.current;result.startTime = new Date(params.startWallTime);this._reporter.onTestBegin?.(test, result);this._currentlyRunning = { test, result };diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex 8ae9b6935d..29292ea420 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -40,6 +40,31 @@ export interface FullResult {* Test duration in milliseconds.*/duration: number;++ /**+ * Only present on merged reports.+ */+ shards?: {+ /**+ * The index of the shard, one-based.+ */+ shardIndex?: number;++ /**+ * Global tag that differentiates CI environments.+ */+ tag: string[];++ /**+ * Start wall time of shard.+ */+ startTime: Date;++ /**+ * Shard run duration in milliseconds.+ */+ duration: number;+ }[];}/**@@ -297,7 +322,7 @@ export interface JSONReportError {export interface JSONReportTestResult {workerIndex: number;parallelIndex: number;- shardIndex: number;+ shardIndex?: number;status: TestStatus | undefined;duration: number;error: TestError | undefined;@@ -682,9 +707,9 @@ export interface TestResult {retry: number;/**- * The index of the shard between `0` and `shards - 1`.+ * The index of the shard between `1` and [`shards`](https://playwright.dev/docs/test-sharding).*/- shardIndex: number;+ shardIndex?: number;/*** Start time of this particular test run.diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex 262f927410..e0d502b797 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -3306,6 +3306,56 @@ for (const useIntermediateMergeReport of [true, false] as const) {await expect(page.getByRole('link', { name: 'previous' })).not.toBeVisible();});});+++ test('shard chart', async ({ runInlineTest, writeFiles, showReport, page }) => {+ test.skip(!useIntermediateMergeReport);++ await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ fullyParallel: true,+ tag: process.env.BOT_TAG,+ };+ `,+ 'a.test.js': `+ import { test, expect } from '@playwright/test';+ import timers from 'timers/promises';+ test('one', async () => {+ await timers.setTimeout(100);+ });+ test('two', async () => {+ await timers.setTimeout(200);+ });+ test('three', async () => {+ await timers.setTimeout(300);+ });+ `,+ });++ await runInlineTest({}, { reporter: 'dot,html', shard: '1/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });+ await runInlineTest({}, { reporter: 'dot,html', shard: '2/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });+ await runInlineTest({}, { reporter: 'dot,html', shard: '3/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' });++ await runInlineTest({}, { reporter: 'dot,html', shard: '1/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });+ await runInlineTest({}, { reporter: 'dot,html', shard: '2/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' });++ await showReport();+ await page.getByRole('link', { name: 'Speedboard' }).click();++ await expect(page.getByRole('main')).toMatchAriaSnapshot(`+ - button "Shard Duration"+ - region:+ - img:+ - list "@linux":+ - listitem /Shard 1/+ - listitem /Shard 2/+ - listitem /Shard 3/+ - list "@mac":+ - listitem /Shard 1/+ - listitem /Shard 2/+ `);+ });});}diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.tsindex c92e37b006..52fd354b04 100644--- a/utils/generate_types/overrides-testReporter.d.ts+++ b/utils/generate_types/overrides-testReporter.d.ts@@ -39,6 +39,31 @@ export interface FullResult {* Test duration in milliseconds.*/duration: number;++ /**+ * Only present on merged reports.+ */+ shards?: {+ /**+ * The index of the shard, one-based.+ */+ shardIndex?: number;++ /**+ * Global tag that differentiates CI environments.+ */+ tag: string[];++ /**+ * Start wall time of shard.+ */+ startTime: Date;++ /**+ * Shard run duration in milliseconds.+ */+ duration: number;+ }[];}export interface Reporter {@@ -110,7 +135,7 @@ export interface JSONReportError {export interface JSONReportTestResult {workerIndex: number;parallelIndex: number;- shardIndex: number;+ shardIndex?: number;status: TestStatus | undefined;duration: number;error: TestError | undefined;