Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: implement shard weights feedback (#38902)
Simon Knott00fb7a962713
diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.ymlindex bce91254e2..ebf04581f1 100644--- a/.github/workflows/tests_primary.yml+++ b/.github/workflows/tests_primary.yml@@ -139,7 +139,7 @@ jobs:- uses: ./.github/actions/run-testwith:node-version: ${{matrix.node-version}}- command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --shard-weights=${{ matrix.shardWeights }}+ command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}bot-name: "${{ matrix.os }}-node${{ matrix.node-version }}"shard-index: ${{ matrix.shardIndex }}flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}@@ -147,6 +147,7 @@ jobs:flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}env:PWTEST_CHANNEL: firefox-beta+ PWTEST_SHARD_WEIGHTS: ${{ matrix.shardWeights }}test_vscode_extension:name: VSCode Extensiondiff --git a/docs/src/images/speedboard-shards.png b/docs/src/images/speedboard-shards.pngdeleted file mode 100644index 3b39ccb2b1..0000000000Binary files a/docs/src/images/speedboard-shards.png and /dev/null differdiff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.mdindex 0d1300b4ad..cd6cf1e643 100644--- a/docs/src/test-reporter-api/class-reporter.md+++ b/docs/src/test-reporter-api/class-reporter.md@@ -132,11 +132,6 @@ Reporter is allowed to override the status and hence affect the exit code of the- `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Test run status.- `startTime` <[Date]> Test run start wall time.- `duration` <[int]> Test run duration in milliseconds.- - `shards` <[Array]<[Object]>> Only present on merged reports- - `shardIndex` ?<[int]> The index of the shard, one-based.- - `tag` ?<[Array]<[string]>> Global [`property: TestConfig.tag`] that differentiates CI environments- - `startTime` <[Date]> Start wall time of shard.- - `duration` <[int]> Shard run duration in milliseconds.Result of the full test run, `status` can be one of:* `'passed'` - Everything went as expected.diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.mdindex eb5f4d7262..ef41200b40 100644--- a/docs/src/test-reporter-api/class-testresult.md+++ b/docs/src/test-reporter-api/class-testresult.md@@ -100,9 +100,3 @@ Learn more about [parallelism and sharding](../test-parallel.md) with Playwright- type: <[int]>The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a different `parallelIndex`.--## property: TestResult.shardIndex-* since: v1.58-- type: ?<[int]>--The index of the shard between `1` and [`shards`](../test-sharding.md).diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.mdindex 56190af17d..f0722a9bd4 100644--- a/docs/src/test-sharding-js.md+++ b/docs/src/test-sharding-js.md@@ -45,21 +45,6 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.-### Rebalancing Shards--If tests in one of your shards take longer, you can manually assign less work to it using the `--shard-weights` option:--```bash-npx playwright test --shard=x/4 --shard-weights=3:2:3:3-```--Make sure you pass the same `--shard-weights` value to all shards.-In this example, Shard 2 gets assigned less tests because they take longer, evening out the full duration.--In the Speedboard section of the combined HTML report, you can see a visualisation of your individual shard durations, and a recommendation for how to set your shard weights.---## Merging reports from multiple shardsIn the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsxdeleted file mode 100644index 105450e94f..0000000000--- a/packages/html-reporter/src/barchart.tsx+++ /dev/null@@ -1,233 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import './barchart.css';--const formatDuration = (ms: number): string => {- const totalSeconds = Math.round(ms / 1000);- const minutes = Math.floor(totalSeconds / 60);- const seconds = totalSeconds % 60;- if (minutes === 0)- return `${seconds}s`;- return `${minutes}m ${seconds}s`;-};--export const GroupedBarChart = ({- data,- groups,- series,-}: {- data: number[][];- groups: string[];- series: string[];-}) => {- const width = 800;-- // Calculate left margin based on longest group name- const maxGroupNameLength = Math.max(...groups.map(g => g.length));- const estimatedTextWidth = maxGroupNameLength * 10;- const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));-- const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };- const chartWidth = width - margin.left - margin.right;-- const maxValue = Math.max(...data.flat());-- let tickInterval: number;- let formatTickLabel: (i: number) => string;-- if (maxValue < 60 * 1000) {- tickInterval = 10 * 1000;- formatTickLabel = i => `${i * 10}s`;- } else if (maxValue < 5 * 60 * 1000) {- tickInterval = 30 * 1000;- formatTickLabel = i => {- const seconds = i * 30;- const mins = Math.floor(seconds / 60);- const secs = seconds % 60;- return secs === 0 ? `${mins}m` : `${mins}m${secs}s`;- };- } else if (maxValue < 30 * 60 * 1000) {- tickInterval = 5 * 60 * 1000;- formatTickLabel = i => `${i * 5}m`;- } else {- tickInterval = 10 * 60 * 1000;- formatTickLabel = i => `${i * 10}m`;- }-- const maxRounded = Math.ceil(maxValue / tickInterval) * tickInterval;- const xScale = chartWidth / maxRounded;-- // Calculate the number of actual bars per group (non-zero values)- const barsPerGroup = data.map(group => group.length);-- // Allocate space proportionally based on number of bars- const barHeight = 20; // Fixed bar height- const barSpacing = 4;- const groupPadding = 20;-- // Calculate Y positions for each group based on their bar count- const groupYPositions: number[] = [];- let currentY = 0;- for (let i = 0; i < groups.length; i++) {- groupYPositions.push(currentY);- const groupHeight = barsPerGroup[i] * barHeight + (barsPerGroup[i] - 1) * barSpacing + groupPadding;- currentY += groupHeight;- }-- const contentHeight = currentY;-- const xTicks = [];- const numberOfTicks = Math.ceil(maxRounded / tickInterval);- for (let i = 0; i <= numberOfTicks; i++) {- xTicks.push({- x: i * tickInterval * xScale,- label: formatTickLabel(i)- });- }-- const height = contentHeight + margin.top + margin.bottom;-- return (- <svg- viewBox={`0 0 ${width} ${height}`}- preserveAspectRatio='xMidYMid meet'- style={{ width: '100%', height: 'auto' }}- role='img'- >- <g transform={`translate(${margin.left}, ${margin.top})`} role='presentation'>- {xTicks.map(({ x, label }, i) => (- <g key={i} aria-hidden='true'>- <line- x1={x}- y1={0}- x2={x}- y2={contentHeight}- stroke='var(--color-border-muted)'- strokeWidth='1'- />- <text- x={x}- y={contentHeight + 20}- textAnchor='middle'- dominantBaseline='middle'- fontSize='12'- fill='var(--color-fg-muted)'- >- {label}- </text>- </g>- ))}-- {groups.map((group, groupIndex) => {- const groupY = groupYPositions[groupIndex];- let barIndex = 0;-- return (- <g key={groupIndex} role='list' aria-label={group}>- {series.map((seriesName, seriesIndex) => {- const value = data[groupIndex][seriesIndex];- if (value === undefined || Number.isNaN(value))- return null;-- const barWidth = value * xScale;- const x = 0;- const y = groupY + barIndex * (barHeight + barSpacing);- barIndex++;-- const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];- const color = colors[seriesIndex % colors.length];-- return (- <g- key={`${groupIndex}-${seriesIndex}`}- role='listitem'- aria-label={`${seriesName}: ${formatDuration(value)}`}- >- <rect- className='barchart-bar'- x={x}- y={y}- width={barWidth}- height={barHeight}- fill={color}- rx='2'- tabIndex={0}- >- <title>{`${seriesName}: ${formatDuration(value)}`}</title>- </rect>- <text- x={barWidth + 6}- y={y + barHeight / 2}- dominantBaseline='middle'- fontSize='12'- fill='var(--color-fg-muted)'- aria-hidden='true'- >- {formatDuration(value)}- </text>- </g>- );- })}- </g>- );- })}-- {groups.map((group, groupIndex) => {- const groupY = groupYPositions[groupIndex];- const actualBars = barsPerGroup[groupIndex];- const groupHeight = actualBars * barHeight + (actualBars - 1) * barSpacing;- const labelY = groupY + groupHeight / 2;-- return (- <text- key={groupIndex}- x={-10}- y={labelY}- textAnchor='end'- dominantBaseline='middle'- fontSize='12'- fill='var(--color-fg-muted)'- aria-hidden='true'- >- {group}- </text>- );- })}-- <line- x1={0}- y1={0}- x2={0}- y2={contentHeight}- stroke='var(--color-fg-muted)'- strokeWidth='1'- aria-hidden='true'- />-- <line- x1={0}- y1={contentHeight}- x2={chartWidth}- y2={contentHeight}- stroke='var(--color-fg-muted)'- strokeWidth='1'- aria-hidden='true'- />- </g>- </svg>- );-};diff --git a/packages/html-reporter/src/barchart.css b/packages/html-reporter/src/gantt.csssimilarity index 92%rename from packages/html-reporter/src/barchart.cssrename to packages/html-reporter/src/gantt.cssindex 08478189a1..85aeb1325d 100644--- a/packages/html-reporter/src/barchart.css+++ b/packages/html-reporter/src/gantt.css@@ -14,14 +14,14 @@limitations under the License.*/-.barchart-bar {+.gantt-bar {transition: opacity 0.2s;cursor: pointer;outline: none;}-.barchart-bar:hover,-.barchart-bar:focus {+.gantt-bar:hover,+.gantt-bar:focus {opacity: 0.8;stroke: var(--color-fg-default);stroke-width: 2;diff --git a/packages/html-reporter/src/gantt.tsx b/packages/html-reporter/src/gantt.tsxnew file mode 100644index 0000000000..de875e8710--- /dev/null+++ b/packages/html-reporter/src/gantt.tsx@@ -0,0 +1,211 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import './gantt.css';++export const formatDuration = (ms: number): string => {+ const totalSeconds = Math.round(ms / 1000);+ const minutes = Math.floor(totalSeconds / 60);+ const seconds = totalSeconds % 60;+ if (minutes === 0)+ return `${seconds}s`;+ return `${minutes}m ${seconds}s`;+};++export type GanttEntry = {+ label: string;+ tooltip: string;+ startTime: number;+ duration: number;+};++export const GanttChart = ({+ entries,+}: {+ entries: GanttEntry[];+}) => {+ const width = 800;++ const maxLabelLength = Math.max(...entries.map(e => e.label.length));+ const estimatedTextWidth = maxLabelLength * 10;+ const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));++ const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };+ const chartWidth = width - margin.left - margin.right;++ const minStartTime = Math.min(...entries.map(e => e.startTime));+ const maxEndTime = Math.max(...entries.map(e => e.startTime + e.duration));++ let tickInterval: number;+ let showSeconds: boolean;+ const duration = maxEndTime - minStartTime;+ if (duration < 60 * 1000) {+ tickInterval = 10 * 1000;+ showSeconds = true;+ } else if (duration < 5 * 60 * 1000) {+ tickInterval = 30 * 1000;+ showSeconds = true;+ } else if (duration < 30 * 60 * 1000) {+ tickInterval = 5 * 60 * 1000;+ showSeconds = false;+ } else {+ tickInterval = 10 * 60 * 1000;+ showSeconds = false;+ }++ // quantize ticks for clean labels+ const firstTick = Math.ceil(minStartTime / tickInterval) * tickInterval;++ const formatTickLabel = (absoluteTime: number, isFirst: boolean) => {+ const label = new Date(absoluteTime).toLocaleTimeString(undefined, {+ hour: '2-digit',+ minute: '2-digit',+ second: showSeconds ? '2-digit' : undefined+ });+ if (isFirst)+ return label;+ // there's no good way of omitting AM/PM other than manually stripping it, "dayPeriod" doesn't work+ if (label.endsWith(' AM') || label.endsWith(' PM'))+ return label.slice(0, -3);+ };++ const maxValue = maxEndTime - minStartTime;+ const maxPadded = maxValue * 1.1;+ const maxRounded = Math.ceil(maxPadded / tickInterval) * tickInterval;+ const xScale = chartWidth / maxRounded;++ const barHeight = 20;+ const barSpacing = 8;++ const contentHeight = entries.length * (barHeight + barSpacing);++ const xTicks = [];+ for (let tickTime = firstTick; tickTime <= minStartTime + maxRounded; tickTime += tickInterval) {+ const tickOffset = tickTime - minStartTime;+ xTicks.push({+ x: tickOffset * xScale,+ label: formatTickLabel(tickTime, tickTime === firstTick)+ });+ }++ const height = contentHeight + margin.top + margin.bottom;++ return (+ <svg+ viewBox={`0 0 ${width} ${height}`}+ preserveAspectRatio='xMidYMid meet'+ style={{ width: '100%', height: 'auto' }}+ role='img'+ >+ <g transform={`translate(${margin.left}, ${margin.top})`} role='presentation'>+ {xTicks.map(({ x, label }, i) => (+ <g key={i} aria-hidden='true'>+ <line+ x1={x}+ y1={0}+ x2={x}+ y2={contentHeight}+ stroke='var(--color-border-muted)'+ strokeWidth='1'+ />+ <text+ x={x}+ y={contentHeight + 20}+ textAnchor='middle'+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ >+ {label}+ </text>+ </g>+ ))}++ {entries.map((entry, index) => {+ const offsetFromStart = entry.startTime - minStartTime;+ const barWidth = entry.duration * xScale;+ const x = offsetFromStart * xScale;+ const y = index * (barHeight + barSpacing);++ const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];+ const color = colors[index % colors.length];++ return (+ <g+ key={index}+ role='listitem'+ aria-label={entry.tooltip}+ >+ <rect+ className='gantt-bar'+ x={x}+ y={y}+ width={barWidth}+ height={barHeight}+ fill={color}+ rx='2'+ tabIndex={0}+ >+ <title>{entry.tooltip}</title>+ </rect>+ <text+ x={x + barWidth + 6}+ y={y + barHeight / 2}+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ aria-hidden='true'+ >+ {formatDuration(entry.duration)}+ </text>+ <text+ x={-10}+ y={y + barHeight / 2}+ textAnchor='end'+ dominantBaseline='middle'+ fontSize='12'+ fill='var(--color-fg-muted)'+ aria-hidden='true'+ >+ {entry.label}+ </text>+ </g>+ );+ })}++ <line+ x1={0}+ y1={0}+ x2={0}+ y2={contentHeight}+ stroke='var(--color-fg-muted)'+ strokeWidth='1'+ aria-hidden='true'+ />++ <line+ x1={0}+ y1={contentHeight}+ x2={chartWidth}+ y2={contentHeight}+ stroke='var(--color-fg-muted)'+ strokeWidth='1'+ aria-hidden='true'+ />+ </g>+ </svg>+ );+};diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsxindex 4f8d84ce4c..f1537df40d 100644--- a/packages/html-reporter/src/speedboard.tsx+++ b/packages/html-reporter/src/speedboard.tsx@@ -20,8 +20,7 @@ import { TestFileView } from './testFileView';import * as icons from './icons';import { TestCaseSummary } from './types';import { AutoChip } from './chip';-import { GroupedBarChart } from './barchart';-import { CodeSnippet } from './testErrorView';+import { formatDuration, GanttChart, GanttEntry } from './gantt';export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {return <>@@ -52,57 +51,28 @@ export function SlowestTests({ report, tests }: { report: LoadedReport, tests: T}export function Shards({ report }: { report: LoadedReport }) {- const shards = report.json().shards;- if (shards.length === 0)+ const machines = report.json().machines;+ if (machines.length === 0)return null;- let clash = false;- const bots: Record<string, { durations: number[], weights: number[] }> = {};- for (const shard of shards) {- const botName = shard.tag.join(' ');- bots[botName] ??= { durations: [], weights: [] };- const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0);- if (bots[botName].durations[shardIndex] !== undefined)- clash = true;- bots[botName].durations[shardIndex] = shard.duration;- bots[botName].weights[shardIndex] = shard.suggestedWeight;- }+ const entries: GanttEntry[] = machines+ .map(machine => {+ const label = machine.tag.join(' ');+ const startTimeFormatted = new Date(machine.startTime).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'short' });+ let tooltip = `${label} started at ${startTimeFormatted}, runs ${formatDuration(machine.duration)}`;+ if (machine.shardIndex)+ tooltip += ` (shard ${machine.shardIndex})`;+ return {+ label,+ tooltip,+ startTime: machine.startTime,+ duration: machine.duration,+ shardIndex: machine.shardIndex ?? 1,+ };+ })+ .sort((a, b) => a.label.localeCompare(b.label) || a.shardIndex - b.shardIndex);- const maxSeries = Math.max(...Object.values(bots).map(b => b.durations.length));- const weightsSnippet = formatWeightCommands(bots);-- return <AutoChip header='Shard Duration'>- <GroupedBarChart- data={Object.values(bots).map(b => b.durations)}- groups={Object.keys(bots)}- series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}- />- {clash && <div style={{ marginTop: 8 }}>- <icons.warning />- Some shards could not be differentiated because of missing global tags.- Please refer to <a href='https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-environments' target='_blank' rel='noopener noreferrer'>- the docs- </a> on how to fix this.- </div>}- {weightsSnippet && <>- <div style={{ marginTop: 8, marginBottom: 16 }}>- Use shard weights to <a href='https://playwright.dev/docs/test-sharding#rebalancing-shards' target='_blank' rel='noopener noreferrer'>- rebalance your shards- </a>:- </div>- <CodeSnippet code={weightsSnippet} />- </>}+ return <AutoChip header='Timeline'>+ <GanttChart entries={entries} /></AutoChip>;}--function formatWeightCommands(bots: Record<string, { weights: number[] }>): string | undefined {- const entries = Object.entries(bots).filter(([, { weights }]) => weights.length > 1);- if (entries.length === 0)- return;-- const maxLen = Math.max(...entries.map(([botName]) => botName.length));- return entries.map(([botName, { weights }]) => {- const prefix = botName ? `${botName}:`.padEnd(maxLen + 2) : '';- return `${prefix}npx playwright test \x1b[33m--shard-weights=\x1b[35m${weights.join(':')}\x1b[39m`;- }).join('\n');-}diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.tsindex 6f288bff80..ddb273bf52 100644--- a/packages/html-reporter/src/types.d.ts+++ b/packages/html-reporter/src/types.d.ts@@ -49,12 +49,11 @@ export type HTMLReport = {projectNames: string[];startTime: number;duration: number;- shards: {+ machines: {shardIndex?: number;tag: string[];startTime: number;duration: number;- suggestedWeight: number;}[];errors: string[]; // Top-level errors that are not attributed to any test.options: HTMLReportOptions;diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.tsindex bae4bde071..dce6cbcd83 100644--- a/packages/playwright/src/common/test.ts+++ b/packages/playwright/src/common/test.ts@@ -345,7 +345,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {retry: this.results.length,parallelIndex: -1,workerIndex: -1,- shardIndex: -1,duration: 0,startTime: new Date(),stdout: [],diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.tsindex c113c8b9b6..b4829257f8 100644--- a/packages/playwright/src/isomorphic/teleReceiver.ts+++ b/packages/playwright/src/isomorphic/teleReceiver.ts@@ -87,7 +87,6 @@ export type JsonTestResultStart = {retry: number;workerIndex: number;parallelIndex: number;- shardIndex?: number;startTime: number;};@@ -130,12 +129,6 @@ export type JsonFullResult = {status: reporterTypes.FullResult['status'];startTime: number;duration: number;- shards?: {- shardIndex?: number;- tag: string[];- startTime: number;- duration: number;- }[];};export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent@@ -362,7 +355,6 @@ export class TeleReporterReceiver {testResult.retry = payload.retry;testResult.workerIndex = payload.workerIndex;testResult.parallelIndex = payload.parallelIndex;- testResult.shardIndex = payload.shardIndex;testResult.setStartTimeNumber(payload.startTime);this._reporter.onTestBegin?.(test, testResult);}@@ -456,12 +448,6 @@ export class TeleReporterReceiver {status: result.status,startTime: new Date(result.startTime),duration: result.duration,- shards: result.shards?.map(s => ({- shardIndex: s.shardIndex,- tag: s.tag,- startTime: new Date(s.startTime),- duration: s.duration,- })) ?? [],});}@@ -735,7 +721,6 @@ export class TeleTestResult implements reporterTypes.TestResult {retry: reporterTypes.TestResult['retry'];parallelIndex: reporterTypes.TestResult['parallelIndex'] = -1;workerIndex: reporterTypes.TestResult['workerIndex'] = -1;- shardIndex: reporterTypes.TestResult['shardIndex'];duration: reporterTypes.TestResult['duration'] = -1;stdout: reporterTypes.TestResult['stdout'] = [];stderr: reporterTypes.TestResult['stderr'] = [];diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex d2f52d3b8d..da4ad0f648 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -300,7 +300,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverridretries: options.retries ? parseInt(options.retries, 10) : undefined,reporter: resolveReporterOption(options.reporter),shard: resolveShardOption(options.shard),- shardWeights: resolveShardWeightsOption(options.shardWeights),+ shardWeights: resolveShardWeightsOption(),timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,@@ -374,14 +374,15 @@ function resolveShardOption(shard?: string): ConfigCLIOverrides['shard'] {return { current, total };}-function resolveShardWeightsOption(shardWeights?: string): ConfigCLIOverrides['shardWeights'] {+function resolveShardWeightsOption(): ConfigCLIOverrides['shardWeights'] {+ const shardWeights = process.env.PWTEST_SHARD_WEIGHTS;if (!shardWeights)return undefined;return shardWeights.split(':').map(w => {const weight = parseInt(w, 10);if (isNaN(weight) || weight < 0)- throw new Error(`--shard-weights "${shardWeights}" weights must be non-negative numbers`);+ throw new Error(`PWTEST_SHARD_WEIGHTS="${shardWeights}" weights must be non-negative numbers`);return weight;});}@@ -425,7 +426,6 @@ const testOptions: [string, { description: string, choices?: string[], preset?:['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],- ['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})` }],diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex 69564b6dc6..c7997fe0ba 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -28,7 +28,7 @@ import { CommonReporterOptions, formatError, formatResultFailure, internalScreenimport { codeFrameColumns } from '../transform/babelBundle';import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';-import type { ReporterV2 } from './reporterV2';+import type { MachineEndResult, ReporterV2 } from './reporterV2';import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test';import type * as api from '../../types/testReporter';import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';@@ -58,6 +58,7 @@ class HtmlReporter implements ReporterV2 {private _host: string | undefined;private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;private _topLevelErrors: api.TestError[] = [];+ private _machines: MachineEndResult[] = [];constructor(options: HtmlReporterConfigOptions & CommonReporterOptions) {this._options = options;@@ -121,6 +122,10 @@ class HtmlReporter implements ReporterV2 {this._topLevelErrors.push(error);}+ onMachineEnd(result: MachineEndResult): void {+ this._machines.push(result);+ }+async onEnd(result: api.FullResult) {const projectSuites = this.suite.suites;await removeFolders([this._outputFolder]);@@ -143,7 +148,7 @@ class HtmlReporter implements ReporterV2 {noSnippets,noCopyPrompt,});- this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors);+ this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines);}async onExit() {@@ -250,7 +255,7 @@ class HtmlBuilder {this._attachmentsBaseURL = attachmentsBaseURL;}- async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {+ async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineEndResult[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {const data: DataMap = new Map();for (const projectSuite of projectSuites) {const projectName = projectSuite.project()!.name;@@ -298,7 +303,12 @@ class HtmlBuilder {stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },errors: topLevelErrors.map(error => formatError(internalScreen, error).message),options: this._options,- shards: this._buildShardsInfo(result, projectSuites),+ machines: machines.map(s => ({+ duration: s.duration,+ startTime: s.startTime.getTime(),+ tag: s.tag,+ shardIndex: s.shardIndex,+ })),};htmlReport.files.sort((f1, f2) => {const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;@@ -342,99 +352,6 @@ class HtmlBuilder {return { ok, singleTestId };}- private _buildShardsInfo(result: api.FullResult, projectSuites: api.Suite[]): HTMLReport['shards'] {- if (!result.shards)- return [];-- const shardsInfo: HTMLReport['shards'] = result.shards.map(s => ({- duration: s.duration,- startTime: s.startTime.getTime(),- tag: s.tag,- shardIndex: s.shardIndex,- suggestedWeight: 100,- }));-- const bots: Record<string, api.TestCase[]> = {};-- for (const projectSuite of projectSuites) {- for (const test of projectSuite.allTests()) {- let longestPrefix: string[] = [];- for (const { tag } of shardsInfo) {- if (longestPrefix.length < tag.length && tag.every((v, i) => test.tags[i] === v))- longestPrefix = tag;- }- const botName = longestPrefix.join(';');- bots[botName] ??= [];- bots[botName].push(test);- }- }-- for (const [botName, tests] of Object.entries(bots)) {- const botShards = shardsInfo.filter(shard => shard.tag.join(';') === botName);- const totalShard = Math.max(...botShards.map(s => s.shardIndex ?? -1));- if (totalShard < 2)- continue;- let suggestedWeights = this._calculateShardWeights(tests, totalShard);- suggestedWeights = this._normaliseWeights(suggestedWeights);- for (const [index, weight] of suggestedWeights.entries()) {- const shard = botShards.find(s => s.shardIndex === index + 1);- if (shard)- shard.suggestedWeight = weight;- }- }-- return shardsInfo;- }-- private _calculateShardWeights(tests: api.TestCase[], shardTotal: number): number[] {- if (tests.length === 0)- return [];-- // recreates the sorting that filterForShard gets as input- tests.sort((a, b) => {- const shardIndexA = a.results[0]!.shardIndex ?? -1;- const shardIndexB = b.results[0]!.shardIndex ?? -1;- if (shardIndexA !== shardIndexB)- return shardIndexA - shardIndexB;-- const startTimeA = a.results[0]!.startTime.getTime();- const startTimeB = b.results[0]!.startTime.getTime();- return startTimeA - startTimeB;- });-- const totalDuration = tests.map(test => test.results[0]!.duration).reduce((a, b) => a + b, 0);- const optimalDuration = totalDuration / shardTotal;-- const weights: number[] = [];- let currentDuration = 0;- let currentTestCount = 0;-- for (const test of tests) {- const duration = test.results[0]!.duration;- currentDuration += duration;- currentTestCount++;-- // When we've reached the optimal shard duration, record this shard's weight- if (currentDuration >= optimalDuration && weights.length < shardTotal - 1) {- weights.push(currentTestCount);- currentDuration = 0;- currentTestCount = 0;- }- }- weights.push(currentTestCount);-- return weights;- }-- private _normaliseWeights(weights: number[]): number[] {- const total = weights.reduce((a, b) => a + b, 0);- weights = weights.map(w => Math.floor((w / total) * 100));- const remaining = 100 - weights.reduce((a, b) => a + b, 0);- for (let i = 0; i < remaining; i++)- weights[i % weights.length]++;- return weights;- }-private async _writeReportData(filePath: string) {fs.appendFileSync(filePath, '<script id="playwrightReportBase64" type="application/zip">data:application/zip;base64,');await new Promise(f => {diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.tsindex e68d34ff25..7b0ac95b82 100644--- a/packages/playwright/src/reporters/internalReporter.ts+++ b/packages/playwright/src/reporters/internalReporter.ts@@ -86,7 +86,6 @@ export class InternalReporter implements ReporterV2 {...result,startTime: this._startTime!,duration: monotonicTime() - this._monotonicStartTime!,- shards: [],});}diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.tsindex 566d06dce1..5823746e50 100644--- a/packages/playwright/src/reporters/json.ts+++ b/packages/playwright/src/reporters/json.ts@@ -200,7 +200,6 @@ class JSONReporter implements ReporterV2 {const jsonResult: JSONReportTestResult = {workerIndex: result.workerIndex,parallelIndex: result.parallelIndex,- shardIndex: result.shardIndex,status: result.status,duration: result.duration,error: result.error,diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.tsindex 55624b0714..8aed8f5280 100644--- a/packages/playwright/src/reporters/merge.ts+++ b/packages/playwright/src/reporters/merge.ts@@ -39,6 +39,8 @@ type ReportData = {reportFile: string;metadata: BlobReportMetadata;tags: string[];+ startTime: number;+ duration: number;};export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {@@ -82,7 +84,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string};await dispatchEvents(eventData.prologue);- for (const { reportFile, eventPatchers, metadata, tags } of eventData.reports) {+ for (const { reportFile, eventPatchers, metadata, tags, startTime, duration } of eventData.reports) {const reportJsonl = await fs.promises.readFile(reportFile);const events = parseTestEvents(reportJsonl);new JsonStringInternalizer(stringPool).traverse(events);@@ -93,6 +95,12 @@ export async function createMergedReport(config: FullConfigInternal, dir: stringeventPatchers.patchers.push(new GlobalErrorPatcher(tags.join(' ')));eventPatchers.patchEvents(events);await dispatchEvents(events);+ multiplexer.onMachineEnd({+ startTime: new Date(startTime),+ duration,+ tag: tags,+ shardIndex: metadata.shard?.current,+ });}await dispatchEvents(eventData.epilogue);}@@ -228,6 +236,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:eventPatchers.patchEvents(parsedEvents);let tags: string[] = [];+ let startTime = 0;+ let duration = 0;for (const event of parsedEvents) {if (event.method === 'onConfigure') {configureEvents.push(event);@@ -236,6 +246,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:projectEvents.push(event);} else if (event.method === 'onEnd') {endEvents.push({ event, metadata, tags });+ startTime = event.params.result.startTime;+ duration = event.params.result.duration;}}@@ -245,6 +257,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:reportFile: localPath,metadata,tags,+ startTime,+ duration,});}@@ -323,13 +337,12 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {};}-function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[]): JsonEvent {+function mergeEndEvents(endEvents: { event: JsonOnEndEvent }[]): JsonEvent {let startTime = endEvents.length ? 10000000000000 : Date.now();let status: JsonFullResult['status'] = 'passed';let endTime: number = 0;- const shards: JsonFullResult['shards'] = [];- for (const { event, metadata, tags } of endEvents) {+ for (const { event } of endEvents) {const shardResult = event.params.result;if (shardResult.status === 'failed')status = 'failed';@@ -339,19 +352,11 @@ function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReportstatus = 'interrupted';startTime = Math.min(startTime, shardResult.startTime);endTime = Math.max(endTime, shardResult.startTime + shardResult.duration);-- shards.push({- shardIndex: metadata.shard?.current,- tag: tags,- startTime: shardResult.startTime,- duration: shardResult.duration,- });}const result: JsonFullResult = {status,startTime,duration: endTime - startTime,- shards,};return {method: 'onEnd',diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.tsindex 43864a7102..d4d2c3f4d5 100644--- a/packages/playwright/src/reporters/multiplexer.ts+++ b/packages/playwright/src/reporters/multiplexer.ts@@ -14,7 +14,7 @@* limitations under the License.*/-import type { ReporterV2 } from './reporterV2';+import type { MachineEndResult, ReporterV2 } from './reporterV2';import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';import type { Suite } from '../common/test';@@ -64,6 +64,11 @@ export class Multiplexer implements ReporterV2 {wrap(() => reporter.onTestEnd?.(test, result));}+ onMachineEnd(result: MachineEndResult): void {+ for (const reporter of this._reporters)+ wrap(() => reporter.onMachineEnd?.(result));+ }+async onEnd(result: FullResult) {for (const reporter of this._reporters) {const outResult = await wrapAsync(() => reporter.onEnd?.(result));diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.tsindex d36050a31f..24507a059e 100644--- a/packages/playwright/src/reporters/reporterV2.ts+++ b/packages/playwright/src/reporters/reporterV2.ts@@ -16,6 +16,13 @@import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';+export interface MachineEndResult {+ tag: string[];+ shardIndex?: number;+ startTime: Date;+ duration: number;+}+export interface ReporterV2 {onConfigure?(config: FullConfig): void;onBegin?(suite: Suite): void;@@ -24,6 +31,7 @@ export interface ReporterV2 {onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;onTestPaused?(test: TestCase, result: TestResult): Promise<void>;onTestEnd?(test: TestCase, result: TestResult): void;+ onMachineEnd?(result: MachineEndResult): void;onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;onExit?(): void | Promise<void>;onError?(error: TestError): void;diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.tsindex 0648706ecb..dd09b91ac9 100644--- a/packages/playwright/src/reporters/teleEmitter.ts+++ b/packages/playwright/src/reporters/teleEmitter.ts@@ -166,12 +166,6 @@ export class TeleReporterEmitter implements ReporterV2 {status: result.status,startTime: result.startTime.getTime(),duration: result.duration,- shards: result.shards?.map(s => ({- shardIndex: s.shardIndex,- tag: s.tag,- startTime: +s.startTime,- duration: s.duration,- })),};this._messageSink({method: 'onEnd',@@ -263,7 +257,6 @@ export class TeleReporterEmitter implements ReporterV2 {retry: result.retry,workerIndex: result.workerIndex,parallelIndex: result.parallelIndex,- shardIndex: result.shardIndex,startTime: +result.startTime,};}diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex db5873fec3..a4008a07aa 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -320,7 +320,6 @@ class JobDispatcher {this._dataByTestId.set(test.id, { test, result, steps: new Map() });result.parallelIndex = this._parallelIndex;result.workerIndex = this._workerIndex;- result.shardIndex = this._config.config.shard?.current;result.startTime = new Date(params.startWallTime);this._reporter.onTestBegin?.(test, result);this._currentlyRunning = { test, result };diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.tsindex c70ebd7586..7ebb2b5b31 100644--- a/packages/playwright/src/runner/testGroups.ts+++ b/packages/playwright/src/runner/testGroups.ts@@ -133,7 +133,7 @@ export function createTestGroups(projectSuite: Suite, expectedParallelism: numbeexport function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {weights ??= Array.from({ length: shard.total }, () => 1);if (weights.length !== shard.total)- throw new Error(`--shard-weights number of weights must match the shard total of ${shard.total}`);+ throw new Error(`PWTEST_SHARD_WEIGHTS number of weights must match the shard total of ${shard.total}`);const totalWeight = weights.reduce((a, b) => a + b, 0);// Note that sharding works based on test groups.diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.tsindex 29292ea420..14154e7769 100644--- a/packages/playwright/types/testReporter.d.ts+++ b/packages/playwright/types/testReporter.d.ts@@ -40,31 +40,6 @@ export interface FullResult {* Test duration in milliseconds.*/duration: number;-- /**- * Only present on merged reports.- */- shards?: {- /**- * The index of the shard, one-based.- */- shardIndex?: number;-- /**- * Global tag that differentiates CI environments.- */- tag: string[];-- /**- * Start wall time of shard.- */- startTime: Date;-- /**- * Shard run duration in milliseconds.- */- duration: number;- }[];}/**@@ -706,11 +681,6 @@ export interface TestResult {*/retry: number;- /**- * The index of the shard between `1` and [`shards`](https://playwright.dev/docs/test-sharding).- */- shardIndex?: number;-/*** Start time of this particular test run.*/diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.tsindex 40cab3922f..779087bbd7 100644--- a/tests/playwright-test/playwright-test-fixtures.ts+++ b/tests/playwright-test/playwright-test-fixtures.ts@@ -237,6 +237,7 @@ export function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {PW_TEST_SOURCE_TRANSFORM: undefined,PW_TEST_SOURCE_TRANSFORM_SCOPE: undefined,PWTEST_BOT_NAME: undefined,+ PWTEST_SHARD_WEIGHTS: undefined,TEST_WORKER_INDEX: undefined,TEST_PARALLEL_INDEX: undefined,NODE_OPTIONS: undefined,diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex c3014f1102..9f4d299887 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -3344,20 +3344,14 @@ for (const useIntermediateMergeReport of [true, false] as const) {await page.getByRole('link', { name: 'Speedboard' }).click();await expect(page.getByRole('main')).toMatchAriaSnapshot(`- - button "Shard Duration"+ - button "Timeline"- region:- img:- - list "@linux":- - listitem /Shard 1/- - listitem /Shard 2/- - listitem /Shard 3/- - list "@mac":- - listitem /Shard 1/- - listitem /Shard 2/- - text: Use shard weights to- - link "rebalance your shards":- - /url: https://playwright.dev/docs/test-sharding#rebalancing-shards- - text: /@linux. npx playwright test --shard-weights=\\d+:\\d+:\\d+ @mac. npx playwright test --shard-weights=\\d+:\\d+/+ - listitem /@linux/+ - listitem /@linux/+ - listitem /@linux/+ - listitem /@mac/+ - listitem /@mac/`);});});diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.tsindex 18f4d52d12..89b995ff35 100644--- a/tests/playwright-test/reporter-json.spec.ts+++ b/tests/playwright-test/reporter-json.spec.ts@@ -68,7 +68,6 @@ test('should not report skipped due to sharding', async ({ runInlineTest }) => {expect(result.report.suites[0].specs.length).toBe(2);expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected');expect(result.report.suites[0].specs[1].tests[0].status).toBe('skipped');- expect(result.report.suites[0].specs[1].tests[0].results[0].shardIndex).toBe(1);});test('should report projects and stats', async ({ runInlineTest }, testInfo) => {diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.tsindex bc3eb2f434..16ef26c251 100644--- a/tests/playwright-test/shard.spec.ts+++ b/tests/playwright-test/shard.spec.ts@@ -335,7 +335,7 @@ test('should shard tests with beforeAll based on shards total instead of workerstest('should respect custom shard weights', async ({ runInlineTest }) => {await test.step('shard 1', async () => {- const result = await runInlineTest(tests, { 'shard': '1/2', 'shard-weights': '40:60', 'workers': 1 });+ const result = await runInlineTest(tests, { 'shard': '1/2', 'workers': 1 }, { PWTEST_SHARD_WEIGHTS: '40:60' });expect.soft(result.exitCode).toBe(0);expect.soft(result.outputLines).toEqual(['a1-test1-done',@@ -345,7 +345,7 @@ test('should respect custom shard weights', async ({ runInlineTest }) => {]);});await test.step('shard 2', async () => {- const result = await runInlineTest(tests, { 'shard': '2/2', 'shard-weights': '40:60', 'workers': 1 });+ const result = await runInlineTest(tests, { 'shard': '2/2', 'workers': 1 }, { PWTEST_SHARD_WEIGHTS: '40:60' });expect.soft(result.exitCode).toBe(0);expect.soft(result.outputLines).toEqual(['a2-test1-done',diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.tsindex 52fd354b04..5a32326fbc 100644--- a/utils/generate_types/overrides-testReporter.d.ts+++ b/utils/generate_types/overrides-testReporter.d.ts@@ -39,31 +39,6 @@ export interface FullResult {* Test duration in milliseconds.*/duration: number;-- /**- * Only present on merged reports.- */- shards?: {- /**- * The index of the shard, one-based.- */- shardIndex?: number;-- /**- * Global tag that differentiates CI environments.- */- tag: string[];-- /**- * Start wall time of shard.- */- startTime: Date;-- /**- * Shard run duration in milliseconds.- */- duration: number;- }[];}export interface Reporter {