Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: implement shard weights feedback (#38902)

Simon Knott00fb7a962713
diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml
index bce91254e2..ebf04581f1 100644
--- a/.github/workflows/tests_primary.yml
+++ b/.github/workflows/tests_primary.yml
@@ -139,7 +139,7 @@ jobs:
- uses: ./.github/actions/run-test
with:
node-version: ${{matrix.node-version}}
- command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --shard-weights=${{ matrix.shardWeights }}
+ command: npm run ttest -- --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
bot-name: "${{ matrix.os }}-node${{ matrix.node-version }}"
shard-index: ${{ matrix.shardIndex }}
flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }}
@@ -147,6 +147,7 @@ jobs:
flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}
env:
PWTEST_CHANNEL: firefox-beta
+ PWTEST_SHARD_WEIGHTS: ${{ matrix.shardWeights }}
test_vscode_extension:
name: VSCode Extension
diff --git a/docs/src/images/speedboard-shards.png b/docs/src/images/speedboard-shards.png
deleted file mode 100644
index 3b39ccb2b1..0000000000
Binary files a/docs/src/images/speedboard-shards.png and /dev/null differ
diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md
index 0d1300b4ad..cd6cf1e643 100644
--- a/docs/src/test-reporter-api/class-reporter.md
+++ b/docs/src/test-reporter-api/class-reporter.md
@@ -132,11 +132,6 @@ Reporter is allowed to override the status and hence affect the exit code of the
- `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Test run status.
- `startTime` <[Date]> Test run start wall time.
- `duration` <[int]> Test run duration in milliseconds.
- - `shards` <[Array]<[Object]>> Only present on merged reports
- - `shardIndex` ?<[int]> The index of the shard, one-based.
- - `tag` ?<[Array]<[string]>> Global [`property: TestConfig.tag`] that differentiates CI environments
- - `startTime` <[Date]> Start wall time of shard.
- - `duration` <[int]> Shard run duration in milliseconds.
Result of the full test run, `status` can be one of:
* `'passed'` - Everything went as expected.
diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md
index eb5f4d7262..ef41200b40 100644
--- a/docs/src/test-reporter-api/class-testresult.md
+++ b/docs/src/test-reporter-api/class-testresult.md
@@ -100,9 +100,3 @@ Learn more about [parallelism and sharding](../test-parallel.md) with Playwright
- type: <[int]>
The index of the worker between `0` and `workers - 1`. It is guaranteed that workers running at the same time have a different `parallelIndex`.
-
-## property: TestResult.shardIndex
-* since: v1.58
-- type: ?<[int]>
-
-The index of the shard between `1` and [`shards`](../test-sharding.md).
diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.md
index 56190af17d..f0722a9bd4 100644
--- a/docs/src/test-sharding-js.md
+++ b/docs/src/test-sharding-js.md
@@ -45,21 +45,6 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul
- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.
- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.
-### Rebalancing Shards
-
-If tests in one of your shards take longer, you can manually assign less work to it using the `--shard-weights` option:
-
-```bash
-npx playwright test --shard=x/4 --shard-weights=3:2:3:3
-```
-
-Make sure you pass the same `--shard-weights` value to all shards.
-In this example, Shard 2 gets assigned less tests because they take longer, evening out the full duration.
-
-In the Speedboard section of the combined HTML report, you can see a visualisation of your individual shard durations, and a recommendation for how to set your shard weights.
-
-![Speedboard Shard Diagram](./images/speedboard-shards.png)
-
## Merging reports from multiple shards
In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.
diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsx
deleted file mode 100644
index 105450e94f..0000000000
--- a/packages/html-reporter/src/barchart.tsx
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import './barchart.css';
-
-const formatDuration = (ms: number): string => {
- const totalSeconds = Math.round(ms / 1000);
- const minutes = Math.floor(totalSeconds / 60);
- const seconds = totalSeconds % 60;
- if (minutes === 0)
- return `${seconds}s`;
- return `${minutes}m ${seconds}s`;
-};
-
-export const GroupedBarChart = ({
- data,
- groups,
- series,
-}: {
- data: number[][];
- groups: string[];
- series: string[];
-}) => {
- const width = 800;
-
- // Calculate left margin based on longest group name
- const maxGroupNameLength = Math.max(...groups.map(g => g.length));
- const estimatedTextWidth = maxGroupNameLength * 10;
- const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));
-
- const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };
- const chartWidth = width - margin.left - margin.right;
-
- const maxValue = Math.max(...data.flat());
-
- let tickInterval: number;
- let formatTickLabel: (i: number) => string;
-
- if (maxValue < 60 * 1000) {
- tickInterval = 10 * 1000;
- formatTickLabel = i => `${i * 10}s`;
- } else if (maxValue < 5 * 60 * 1000) {
- tickInterval = 30 * 1000;
- formatTickLabel = i => {
- const seconds = i * 30;
- const mins = Math.floor(seconds / 60);
- const secs = seconds % 60;
- return secs === 0 ? `${mins}m` : `${mins}m${secs}s`;
- };
- } else if (maxValue < 30 * 60 * 1000) {
- tickInterval = 5 * 60 * 1000;
- formatTickLabel = i => `${i * 5}m`;
- } else {
- tickInterval = 10 * 60 * 1000;
- formatTickLabel = i => `${i * 10}m`;
- }
-
- const maxRounded = Math.ceil(maxValue / tickInterval) * tickInterval;
- const xScale = chartWidth / maxRounded;
-
- // Calculate the number of actual bars per group (non-zero values)
- const barsPerGroup = data.map(group => group.length);
-
- // Allocate space proportionally based on number of bars
- const barHeight = 20; // Fixed bar height
- const barSpacing = 4;
- const groupPadding = 20;
-
- // Calculate Y positions for each group based on their bar count
- const groupYPositions: number[] = [];
- let currentY = 0;
- for (let i = 0; i < groups.length; i++) {
- groupYPositions.push(currentY);
- const groupHeight = barsPerGroup[i] * barHeight + (barsPerGroup[i] - 1) * barSpacing + groupPadding;
- currentY += groupHeight;
- }
-
- const contentHeight = currentY;
-
- const xTicks = [];
- const numberOfTicks = Math.ceil(maxRounded / tickInterval);
- for (let i = 0; i <= numberOfTicks; i++) {
- xTicks.push({
- x: i * tickInterval * xScale,
- label: formatTickLabel(i)
- });
- }
-
- const height = contentHeight + margin.top + margin.bottom;
-
- return (
- <svg
- viewBox={`0 0 ${width} ${height}`}
- preserveAspectRatio='xMidYMid meet'
- style={{ width: '100%', height: 'auto' }}
- role='img'
- >
- <g transform={`translate(${margin.left}, ${margin.top})`} role='presentation'>
- {xTicks.map(({ x, label }, i) => (
- <g key={i} aria-hidden='true'>
- <line
- x1={x}
- y1={0}
- x2={x}
- y2={contentHeight}
- stroke='var(--color-border-muted)'
- strokeWidth='1'
- />
- <text
- x={x}
- y={contentHeight + 20}
- textAnchor='middle'
- dominantBaseline='middle'
- fontSize='12'
- fill='var(--color-fg-muted)'
- >
- {label}
- </text>
- </g>
- ))}
-
- {groups.map((group, groupIndex) => {
- const groupY = groupYPositions[groupIndex];
- let barIndex = 0;
-
- return (
- <g key={groupIndex} role='list' aria-label={group}>
- {series.map((seriesName, seriesIndex) => {
- const value = data[groupIndex][seriesIndex];
- if (value === undefined || Number.isNaN(value))
- return null;
-
- const barWidth = value * xScale;
- const x = 0;
- const y = groupY + barIndex * (barHeight + barSpacing);
- barIndex++;
-
- const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];
- const color = colors[seriesIndex % colors.length];
-
- return (
- <g
- key={`${groupIndex}-${seriesIndex}`}
- role='listitem'
- aria-label={`${seriesName}: ${formatDuration(value)}`}
- >
- <rect
- className='barchart-bar'
- x={x}
- y={y}
- width={barWidth}
- height={barHeight}
- fill={color}
- rx='2'
- tabIndex={0}
- >
- <title>{`${seriesName}: ${formatDuration(value)}`}</title>
- </rect>
- <text
- x={barWidth + 6}
- y={y + barHeight / 2}
- dominantBaseline='middle'
- fontSize='12'
- fill='var(--color-fg-muted)'
- aria-hidden='true'
- >
- {formatDuration(value)}
- </text>
- </g>
- );
- })}
- </g>
- );
- })}
-
- {groups.map((group, groupIndex) => {
- const groupY = groupYPositions[groupIndex];
- const actualBars = barsPerGroup[groupIndex];
- const groupHeight = actualBars * barHeight + (actualBars - 1) * barSpacing;
- const labelY = groupY + groupHeight / 2;
-
- return (
- <text
- key={groupIndex}
- x={-10}
- y={labelY}
- textAnchor='end'
- dominantBaseline='middle'
- fontSize='12'
- fill='var(--color-fg-muted)'
- aria-hidden='true'
- >
- {group}
- </text>
- );
- })}
-
- <line
- x1={0}
- y1={0}
- x2={0}
- y2={contentHeight}
- stroke='var(--color-fg-muted)'
- strokeWidth='1'
- aria-hidden='true'
- />
-
- <line
- x1={0}
- y1={contentHeight}
- x2={chartWidth}
- y2={contentHeight}
- stroke='var(--color-fg-muted)'
- strokeWidth='1'
- aria-hidden='true'
- />
- </g>
- </svg>
- );
-};
diff --git a/packages/html-reporter/src/barchart.css b/packages/html-reporter/src/gantt.css
similarity index 92%
rename from packages/html-reporter/src/barchart.css
rename to packages/html-reporter/src/gantt.css
index 08478189a1..85aeb1325d 100644
--- a/packages/html-reporter/src/barchart.css
+++ b/packages/html-reporter/src/gantt.css
@@ -14,14 +14,14 @@
limitations under the License.
*/
-.barchart-bar {
+.gantt-bar {
transition: opacity 0.2s;
cursor: pointer;
outline: none;
}
-.barchart-bar:hover,
-.barchart-bar:focus {
+.gantt-bar:hover,
+.gantt-bar:focus {
opacity: 0.8;
stroke: var(--color-fg-default);
stroke-width: 2;
diff --git a/packages/html-reporter/src/gantt.tsx b/packages/html-reporter/src/gantt.tsx
new file mode 100644
index 0000000000..de875e8710
--- /dev/null
+++ b/packages/html-reporter/src/gantt.tsx
@@ -0,0 +1,211 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './gantt.css';
+
+export const formatDuration = (ms: number): string => {
+ const totalSeconds = Math.round(ms / 1000);
+ const minutes = Math.floor(totalSeconds / 60);
+ const seconds = totalSeconds % 60;
+ if (minutes === 0)
+ return `${seconds}s`;
+ return `${minutes}m ${seconds}s`;
+};
+
+export type GanttEntry = {
+ label: string;
+ tooltip: string;
+ startTime: number;
+ duration: number;
+};
+
+export const GanttChart = ({
+ entries,
+}: {
+ entries: GanttEntry[];
+}) => {
+ const width = 800;
+
+ const maxLabelLength = Math.max(...entries.map(e => e.label.length));
+ const estimatedTextWidth = maxLabelLength * 10;
+ const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));
+
+ const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };
+ const chartWidth = width - margin.left - margin.right;
+
+ const minStartTime = Math.min(...entries.map(e => e.startTime));
+ const maxEndTime = Math.max(...entries.map(e => e.startTime + e.duration));
+
+ let tickInterval: number;
+ let showSeconds: boolean;
+ const duration = maxEndTime - minStartTime;
+ if (duration < 60 * 1000) {
+ tickInterval = 10 * 1000;
+ showSeconds = true;
+ } else if (duration < 5 * 60 * 1000) {
+ tickInterval = 30 * 1000;
+ showSeconds = true;
+ } else if (duration < 30 * 60 * 1000) {
+ tickInterval = 5 * 60 * 1000;
+ showSeconds = false;
+ } else {
+ tickInterval = 10 * 60 * 1000;
+ showSeconds = false;
+ }
+
+ // quantize ticks for clean labels
+ const firstTick = Math.ceil(minStartTime / tickInterval) * tickInterval;
+
+ const formatTickLabel = (absoluteTime: number, isFirst: boolean) => {
+ const label = new Date(absoluteTime).toLocaleTimeString(undefined, {
+ hour: '2-digit',
+ minute: '2-digit',
+ second: showSeconds ? '2-digit' : undefined
+ });
+ if (isFirst)
+ return label;
+ // there's no good way of omitting AM/PM other than manually stripping it, "dayPeriod" doesn't work
+ if (label.endsWith(' AM') || label.endsWith(' PM'))
+ return label.slice(0, -3);
+ };
+
+ const maxValue = maxEndTime - minStartTime;
+ const maxPadded = maxValue * 1.1;
+ const maxRounded = Math.ceil(maxPadded / tickInterval) * tickInterval;
+ const xScale = chartWidth / maxRounded;
+
+ const barHeight = 20;
+ const barSpacing = 8;
+
+ const contentHeight = entries.length * (barHeight + barSpacing);
+
+ const xTicks = [];
+ for (let tickTime = firstTick; tickTime <= minStartTime + maxRounded; tickTime += tickInterval) {
+ const tickOffset = tickTime - minStartTime;
+ xTicks.push({
+ x: tickOffset * xScale,
+ label: formatTickLabel(tickTime, tickTime === firstTick)
+ });
+ }
+
+ const height = contentHeight + margin.top + margin.bottom;
+
+ return (
+ <svg
+ viewBox={`0 0 ${width} ${height}`}
+ preserveAspectRatio='xMidYMid meet'
+ style={{ width: '100%', height: 'auto' }}
+ role='img'
+ >
+ <g transform={`translate(${margin.left}, ${margin.top})`} role='presentation'>
+ {xTicks.map(({ x, label }, i) => (
+ <g key={i} aria-hidden='true'>
+ <line
+ x1={x}
+ y1={0}
+ x2={x}
+ y2={contentHeight}
+ stroke='var(--color-border-muted)'
+ strokeWidth='1'
+ />
+ <text
+ x={x}
+ y={contentHeight + 20}
+ textAnchor='middle'
+ dominantBaseline='middle'
+ fontSize='12'
+ fill='var(--color-fg-muted)'
+ >
+ {label}
+ </text>
+ </g>
+ ))}
+
+ {entries.map((entry, index) => {
+ const offsetFromStart = entry.startTime - minStartTime;
+ const barWidth = entry.duration * xScale;
+ const x = offsetFromStart * xScale;
+ const y = index * (barHeight + barSpacing);
+
+ const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];
+ const color = colors[index % colors.length];
+
+ return (
+ <g
+ key={index}
+ role='listitem'
+ aria-label={entry.tooltip}
+ >
+ <rect
+ className='gantt-bar'
+ x={x}
+ y={y}
+ width={barWidth}
+ height={barHeight}
+ fill={color}
+ rx='2'
+ tabIndex={0}
+ >
+ <title>{entry.tooltip}</title>
+ </rect>
+ <text
+ x={x + barWidth + 6}
+ y={y + barHeight / 2}
+ dominantBaseline='middle'
+ fontSize='12'
+ fill='var(--color-fg-muted)'
+ aria-hidden='true'
+ >
+ {formatDuration(entry.duration)}
+ </text>
+ <text
+ x={-10}
+ y={y + barHeight / 2}
+ textAnchor='end'
+ dominantBaseline='middle'
+ fontSize='12'
+ fill='var(--color-fg-muted)'
+ aria-hidden='true'
+ >
+ {entry.label}
+ </text>
+ </g>
+ );
+ })}
+
+ <line
+ x1={0}
+ y1={0}
+ x2={0}
+ y2={contentHeight}
+ stroke='var(--color-fg-muted)'
+ strokeWidth='1'
+ aria-hidden='true'
+ />
+
+ <line
+ x1={0}
+ y1={contentHeight}
+ x2={chartWidth}
+ y2={contentHeight}
+ stroke='var(--color-fg-muted)'
+ strokeWidth='1'
+ aria-hidden='true'
+ />
+ </g>
+ </svg>
+ );
+};
diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsx
index 4f8d84ce4c..f1537df40d 100644
--- a/packages/html-reporter/src/speedboard.tsx
+++ b/packages/html-reporter/src/speedboard.tsx
@@ -20,8 +20,7 @@ import { TestFileView } from './testFileView';
import * as icons from './icons';
import { TestCaseSummary } from './types';
import { AutoChip } from './chip';
-import { GroupedBarChart } from './barchart';
-import { CodeSnippet } from './testErrorView';
+import { formatDuration, GanttChart, GanttEntry } from './gantt';
export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {
return <>
@@ -52,57 +51,28 @@ export function SlowestTests({ report, tests }: { report: LoadedReport, tests: T
}
export function Shards({ report }: { report: LoadedReport }) {
- const shards = report.json().shards;
- if (shards.length === 0)
+ const machines = report.json().machines;
+ if (machines.length === 0)
return null;
- let clash = false;
- const bots: Record<string, { durations: number[], weights: number[] }> = {};
- for (const shard of shards) {
- const botName = shard.tag.join(' ');
- bots[botName] ??= { durations: [], weights: [] };
- const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0);
- if (bots[botName].durations[shardIndex] !== undefined)
- clash = true;
- bots[botName].durations[shardIndex] = shard.duration;
- bots[botName].weights[shardIndex] = shard.suggestedWeight;
- }
+ const entries: GanttEntry[] = machines
+ .map(machine => {
+ const label = machine.tag.join(' ');
+ const startTimeFormatted = new Date(machine.startTime).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'short' });
+ let tooltip = `${label} started at ${startTimeFormatted}, runs ${formatDuration(machine.duration)}`;
+ if (machine.shardIndex)
+ tooltip += ` (shard ${machine.shardIndex})`;
+ return {
+ label,
+ tooltip,
+ startTime: machine.startTime,
+ duration: machine.duration,
+ shardIndex: machine.shardIndex ?? 1,
+ };
+ })
+ .sort((a, b) => a.label.localeCompare(b.label) || a.shardIndex - b.shardIndex);
- const maxSeries = Math.max(...Object.values(bots).map(b => b.durations.length));
- const weightsSnippet = formatWeightCommands(bots);
-
- return <AutoChip header='Shard Duration'>
- <GroupedBarChart
- data={Object.values(bots).map(b => b.durations)}
- groups={Object.keys(bots)}
- series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}
- />
- {clash && <div style={{ marginTop: 8 }}>
- <icons.warning />
- Some shards could not be differentiated because of missing global tags.
- Please refer to <a href='https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-environments' target='_blank' rel='noopener noreferrer'>
- the docs
- </a> on how to fix this.
- </div>}
- {weightsSnippet && <>
- <div style={{ marginTop: 8, marginBottom: 16 }}>
- Use shard weights to <a href='https://playwright.dev/docs/test-sharding#rebalancing-shards' target='_blank' rel='noopener noreferrer'>
- rebalance your shards
- </a>:
- </div>
- <CodeSnippet code={weightsSnippet} />
- </>}
+ return <AutoChip header='Timeline'>
+ <GanttChart entries={entries} />
</AutoChip>;
}
-
-function formatWeightCommands(bots: Record<string, { weights: number[] }>): string | undefined {
- const entries = Object.entries(bots).filter(([, { weights }]) => weights.length > 1);
- if (entries.length === 0)
- return;
-
- const maxLen = Math.max(...entries.map(([botName]) => botName.length));
- return entries.map(([botName, { weights }]) => {
- const prefix = botName ? `${botName}:`.padEnd(maxLen + 2) : '';
- return `${prefix}npx playwright test \x1b[33m--shard-weights=\x1b[35m${weights.join(':')}\x1b[39m`;
- }).join('\n');
-}
diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts
index 6f288bff80..ddb273bf52 100644
--- a/packages/html-reporter/src/types.d.ts
+++ b/packages/html-reporter/src/types.d.ts
@@ -49,12 +49,11 @@ export type HTMLReport = {
projectNames: string[];
startTime: number;
duration: number;
- shards: {
+ machines: {
shardIndex?: number;
tag: string[];
startTime: number;
duration: number;
- suggestedWeight: number;
}[];
errors: string[]; // Top-level errors that are not attributed to any test.
options: HTMLReportOptions;
diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts
index bae4bde071..dce6cbcd83 100644
--- a/packages/playwright/src/common/test.ts
+++ b/packages/playwright/src/common/test.ts
@@ -345,7 +345,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
retry: this.results.length,
parallelIndex: -1,
workerIndex: -1,
- shardIndex: -1,
duration: 0,
startTime: new Date(),
stdout: [],
diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts
index c113c8b9b6..b4829257f8 100644
--- a/packages/playwright/src/isomorphic/teleReceiver.ts
+++ b/packages/playwright/src/isomorphic/teleReceiver.ts
@@ -87,7 +87,6 @@ export type JsonTestResultStart = {
retry: number;
workerIndex: number;
parallelIndex: number;
- shardIndex?: number;
startTime: number;
};
@@ -130,12 +129,6 @@ export type JsonFullResult = {
status: reporterTypes.FullResult['status'];
startTime: number;
duration: number;
- shards?: {
- shardIndex?: number;
- tag: string[];
- startTime: number;
- duration: number;
- }[];
};
export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
@@ -362,7 +355,6 @@ export class TeleReporterReceiver {
testResult.retry = payload.retry;
testResult.workerIndex = payload.workerIndex;
testResult.parallelIndex = payload.parallelIndex;
- testResult.shardIndex = payload.shardIndex;
testResult.setStartTimeNumber(payload.startTime);
this._reporter.onTestBegin?.(test, testResult);
}
@@ -456,12 +448,6 @@ export class TeleReporterReceiver {
status: result.status,
startTime: new Date(result.startTime),
duration: result.duration,
- shards: result.shards?.map(s => ({
- shardIndex: s.shardIndex,
- tag: s.tag,
- startTime: new Date(s.startTime),
- duration: s.duration,
- })) ?? [],
});
}
@@ -735,7 +721,6 @@ export class TeleTestResult implements reporterTypes.TestResult {
retry: reporterTypes.TestResult['retry'];
parallelIndex: reporterTypes.TestResult['parallelIndex'] = -1;
workerIndex: reporterTypes.TestResult['workerIndex'] = -1;
- shardIndex: reporterTypes.TestResult['shardIndex'];
duration: reporterTypes.TestResult['duration'] = -1;
stdout: reporterTypes.TestResult['stdout'] = [];
stderr: reporterTypes.TestResult['stderr'] = [];
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index d2f52d3b8d..da4ad0f648 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -300,7 +300,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
retries: options.retries ? parseInt(options.retries, 10) : undefined,
reporter: resolveReporterOption(options.reporter),
shard: resolveShardOption(options.shard),
- shardWeights: resolveShardWeightsOption(options.shardWeights),
+ shardWeights: resolveShardWeightsOption(),
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
tsconfig: options.tsconfig ? path.resolve(process.cwd(), options.tsconfig) : undefined,
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -374,14 +374,15 @@ function resolveShardOption(shard?: string): ConfigCLIOverrides['shard'] {
return { current, total };
}
-function resolveShardWeightsOption(shardWeights?: string): ConfigCLIOverrides['shardWeights'] {
+function resolveShardWeightsOption(): ConfigCLIOverrides['shardWeights'] {
+ const shardWeights = process.env.PWTEST_SHARD_WEIGHTS;
if (!shardWeights)
return undefined;
return shardWeights.split(':').map(w => {
const weight = parseInt(w, 10);
if (isNaN(weight) || weight < 0)
- throw new Error(`--shard-weights "${shardWeights}" weights must be non-negative numbers`);
+ throw new Error(`PWTEST_SHARD_WEIGHTS="${shardWeights}" weights must be non-negative numbers`);
return weight;
});
}
@@ -425,7 +426,6 @@ const testOptions: [string, { description: string, choices?: string[], preset?:
['--retries <retries>', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }],
['--run-agents <mode>', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }],
['--shard <shard>', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }],
- ['--shard-weights <weights>', { description: `Weights for each shard, colon-separated, for example "2:1:1" for 3 shards where the first shard should be allocated half of the work` }],
['--test-list <file>', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }],
['--test-list-invert <file>', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }],
['--timeout <timeout>', { description: `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})` }],
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index 69564b6dc6..c7997fe0ba 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -28,7 +28,7 @@ import { CommonReporterOptions, formatError, formatResultFailure, internalScreen
import { codeFrameColumns } from '../transform/babelBundle';
import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';
-import type { ReporterV2 } from './reporterV2';
+import type { MachineEndResult, ReporterV2 } from './reporterV2';
import type { HtmlReporterOptions as HtmlReporterConfigOptions, Metadata, TestAnnotation } from '../../types/test';
import type * as api from '../../types/testReporter';
import type { HTMLReport, HTMLReportOptions, Location, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';
@@ -58,6 +58,7 @@ class HtmlReporter implements ReporterV2 {
private _host: string | undefined;
private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
private _topLevelErrors: api.TestError[] = [];
+ private _machines: MachineEndResult[] = [];
constructor(options: HtmlReporterConfigOptions & CommonReporterOptions) {
this._options = options;
@@ -121,6 +122,10 @@ class HtmlReporter implements ReporterV2 {
this._topLevelErrors.push(error);
}
+ onMachineEnd(result: MachineEndResult): void {
+ this._machines.push(result);
+ }
+
async onEnd(result: api.FullResult) {
const projectSuites = this.suite.suites;
await removeFolders([this._outputFolder]);
@@ -143,7 +148,7 @@ class HtmlReporter implements ReporterV2 {
noSnippets,
noCopyPrompt,
});
- this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors);
+ this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors, this._machines);
}
async onExit() {
@@ -250,7 +255,7 @@ class HtmlBuilder {
this._attachmentsBaseURL = attachmentsBaseURL;
}
- async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+ async build(metadata: Metadata, projectSuites: api.Suite[], result: api.FullResult, topLevelErrors: api.TestError[], machines: MachineEndResult[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
const data: DataMap = new Map();
for (const projectSuite of projectSuites) {
const projectName = projectSuite.project()!.name;
@@ -298,7 +303,12 @@ class HtmlBuilder {
stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },
errors: topLevelErrors.map(error => formatError(internalScreen, error).message),
options: this._options,
- shards: this._buildShardsInfo(result, projectSuites),
+ machines: machines.map(s => ({
+ duration: s.duration,
+ startTime: s.startTime.getTime(),
+ tag: s.tag,
+ shardIndex: s.shardIndex,
+ })),
};
htmlReport.files.sort((f1, f2) => {
const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;
@@ -342,99 +352,6 @@ class HtmlBuilder {
return { ok, singleTestId };
}
- private _buildShardsInfo(result: api.FullResult, projectSuites: api.Suite[]): HTMLReport['shards'] {
- if (!result.shards)
- return [];
-
- const shardsInfo: HTMLReport['shards'] = result.shards.map(s => ({
- duration: s.duration,
- startTime: s.startTime.getTime(),
- tag: s.tag,
- shardIndex: s.shardIndex,
- suggestedWeight: 100,
- }));
-
- const bots: Record<string, api.TestCase[]> = {};
-
- for (const projectSuite of projectSuites) {
- for (const test of projectSuite.allTests()) {
- let longestPrefix: string[] = [];
- for (const { tag } of shardsInfo) {
- if (longestPrefix.length < tag.length && tag.every((v, i) => test.tags[i] === v))
- longestPrefix = tag;
- }
- const botName = longestPrefix.join(';');
- bots[botName] ??= [];
- bots[botName].push(test);
- }
- }
-
- for (const [botName, tests] of Object.entries(bots)) {
- const botShards = shardsInfo.filter(shard => shard.tag.join(';') === botName);
- const totalShard = Math.max(...botShards.map(s => s.shardIndex ?? -1));
- if (totalShard < 2)
- continue;
- let suggestedWeights = this._calculateShardWeights(tests, totalShard);
- suggestedWeights = this._normaliseWeights(suggestedWeights);
- for (const [index, weight] of suggestedWeights.entries()) {
- const shard = botShards.find(s => s.shardIndex === index + 1);
- if (shard)
- shard.suggestedWeight = weight;
- }
- }
-
- return shardsInfo;
- }
-
- private _calculateShardWeights(tests: api.TestCase[], shardTotal: number): number[] {
- if (tests.length === 0)
- return [];
-
- // recreates the sorting that filterForShard gets as input
- tests.sort((a, b) => {
- const shardIndexA = a.results[0]!.shardIndex ?? -1;
- const shardIndexB = b.results[0]!.shardIndex ?? -1;
- if (shardIndexA !== shardIndexB)
- return shardIndexA - shardIndexB;
-
- const startTimeA = a.results[0]!.startTime.getTime();
- const startTimeB = b.results[0]!.startTime.getTime();
- return startTimeA - startTimeB;
- });
-
- const totalDuration = tests.map(test => test.results[0]!.duration).reduce((a, b) => a + b, 0);
- const optimalDuration = totalDuration / shardTotal;
-
- const weights: number[] = [];
- let currentDuration = 0;
- let currentTestCount = 0;
-
- for (const test of tests) {
- const duration = test.results[0]!.duration;
- currentDuration += duration;
- currentTestCount++;
-
- // When we've reached the optimal shard duration, record this shard's weight
- if (currentDuration >= optimalDuration && weights.length < shardTotal - 1) {
- weights.push(currentTestCount);
- currentDuration = 0;
- currentTestCount = 0;
- }
- }
- weights.push(currentTestCount);
-
- return weights;
- }
-
- private _normaliseWeights(weights: number[]): number[] {
- const total = weights.reduce((a, b) => a + b, 0);
- weights = weights.map(w => Math.floor((w / total) * 100));
- const remaining = 100 - weights.reduce((a, b) => a + b, 0);
- for (let i = 0; i < remaining; i++)
- weights[i % weights.length]++;
- return weights;
- }
-
private async _writeReportData(filePath: string) {
fs.appendFileSync(filePath, '<script id="playwrightReportBase64" type="application/zip">data:application/zip;base64,');
await new Promise(f => {
diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts
index e68d34ff25..7b0ac95b82 100644
--- a/packages/playwright/src/reporters/internalReporter.ts
+++ b/packages/playwright/src/reporters/internalReporter.ts
@@ -86,7 +86,6 @@ export class InternalReporter implements ReporterV2 {
...result,
startTime: this._startTime!,
duration: monotonicTime() - this._monotonicStartTime!,
- shards: [],
});
}
diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts
index 566d06dce1..5823746e50 100644
--- a/packages/playwright/src/reporters/json.ts
+++ b/packages/playwright/src/reporters/json.ts
@@ -200,7 +200,6 @@ class JSONReporter implements ReporterV2 {
const jsonResult: JSONReportTestResult = {
workerIndex: result.workerIndex,
parallelIndex: result.parallelIndex,
- shardIndex: result.shardIndex,
status: result.status,
duration: result.duration,
error: result.error,
diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts
index 55624b0714..8aed8f5280 100644
--- a/packages/playwright/src/reporters/merge.ts
+++ b/packages/playwright/src/reporters/merge.ts
@@ -39,6 +39,8 @@ type ReportData = {
reportFile: string;
metadata: BlobReportMetadata;
tags: string[];
+ startTime: number;
+ duration: number;
};
export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], rootDirOverride: string | undefined) {
@@ -82,7 +84,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
};
await dispatchEvents(eventData.prologue);
- for (const { reportFile, eventPatchers, metadata, tags } of eventData.reports) {
+ for (const { reportFile, eventPatchers, metadata, tags, startTime, duration } of eventData.reports) {
const reportJsonl = await fs.promises.readFile(reportFile);
const events = parseTestEvents(reportJsonl);
new JsonStringInternalizer(stringPool).traverse(events);
@@ -93,6 +95,12 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
eventPatchers.patchers.push(new GlobalErrorPatcher(tags.join(' ')));
eventPatchers.patchEvents(events);
await dispatchEvents(events);
+ multiplexer.onMachineEnd({
+ startTime: new Date(startTime),
+ duration,
+ tag: tags,
+ shardIndex: metadata.shard?.current,
+ });
}
await dispatchEvents(eventData.epilogue);
}
@@ -228,6 +236,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
eventPatchers.patchEvents(parsedEvents);
let tags: string[] = [];
+ let startTime = 0;
+ let duration = 0;
for (const event of parsedEvents) {
if (event.method === 'onConfigure') {
configureEvents.push(event);
@@ -236,6 +246,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
projectEvents.push(event);
} else if (event.method === 'onEnd') {
endEvents.push({ event, metadata, tags });
+ startTime = event.params.result.startTime;
+ duration = event.params.result.duration;
}
}
@@ -245,6 +257,8 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
reportFile: localPath,
metadata,
tags,
+ startTime,
+ duration,
});
}
@@ -323,13 +337,12 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {
};
}
-function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[]): JsonEvent {
+function mergeEndEvents(endEvents: { event: JsonOnEndEvent }[]): JsonEvent {
let startTime = endEvents.length ? 10000000000000 : Date.now();
let status: JsonFullResult['status'] = 'passed';
let endTime: number = 0;
- const shards: JsonFullResult['shards'] = [];
- for (const { event, metadata, tags } of endEvents) {
+ for (const { event } of endEvents) {
const shardResult = event.params.result;
if (shardResult.status === 'failed')
status = 'failed';
@@ -339,19 +352,11 @@ function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReport
status = 'interrupted';
startTime = Math.min(startTime, shardResult.startTime);
endTime = Math.max(endTime, shardResult.startTime + shardResult.duration);
-
- shards.push({
- shardIndex: metadata.shard?.current,
- tag: tags,
- startTime: shardResult.startTime,
- duration: shardResult.duration,
- });
}
const result: JsonFullResult = {
status,
startTime,
duration: endTime - startTime,
- shards,
};
return {
method: 'onEnd',
diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts
index 43864a7102..d4d2c3f4d5 100644
--- a/packages/playwright/src/reporters/multiplexer.ts
+++ b/packages/playwright/src/reporters/multiplexer.ts
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-import type { ReporterV2 } from './reporterV2';
+import type { MachineEndResult, ReporterV2 } from './reporterV2';
import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
import type { Suite } from '../common/test';
@@ -64,6 +64,11 @@ export class Multiplexer implements ReporterV2 {
wrap(() => reporter.onTestEnd?.(test, result));
}
+ onMachineEnd(result: MachineEndResult): void {
+ for (const reporter of this._reporters)
+ wrap(() => reporter.onMachineEnd?.(result));
+ }
+
async onEnd(result: FullResult) {
for (const reporter of this._reporters) {
const outResult = await wrapAsync(() => reporter.onEnd?.(result));
diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts
index d36050a31f..24507a059e 100644
--- a/packages/playwright/src/reporters/reporterV2.ts
+++ b/packages/playwright/src/reporters/reporterV2.ts
@@ -16,6 +16,13 @@
import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
+export interface MachineEndResult {
+ tag: string[];
+ shardIndex?: number;
+ startTime: Date;
+ duration: number;
+}
+
export interface ReporterV2 {
onConfigure?(config: FullConfig): void;
onBegin?(suite: Suite): void;
@@ -24,6 +31,7 @@ export interface ReporterV2 {
onStdErr?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
onTestPaused?(test: TestCase, result: TestResult): Promise<void>;
onTestEnd?(test: TestCase, result: TestResult): void;
+ onMachineEnd?(result: MachineEndResult): void;
onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
onExit?(): void | Promise<void>;
onError?(error: TestError): void;
diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts
index 0648706ecb..dd09b91ac9 100644
--- a/packages/playwright/src/reporters/teleEmitter.ts
+++ b/packages/playwright/src/reporters/teleEmitter.ts
@@ -166,12 +166,6 @@ export class TeleReporterEmitter implements ReporterV2 {
status: result.status,
startTime: result.startTime.getTime(),
duration: result.duration,
- shards: result.shards?.map(s => ({
- shardIndex: s.shardIndex,
- tag: s.tag,
- startTime: +s.startTime,
- duration: s.duration,
- })),
};
this._messageSink({
method: 'onEnd',
@@ -263,7 +257,6 @@ export class TeleReporterEmitter implements ReporterV2 {
retry: result.retry,
workerIndex: result.workerIndex,
parallelIndex: result.parallelIndex,
- shardIndex: result.shardIndex,
startTime: +result.startTime,
};
}
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index db5873fec3..a4008a07aa 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -320,7 +320,6 @@ class JobDispatcher {
this._dataByTestId.set(test.id, { test, result, steps: new Map() });
result.parallelIndex = this._parallelIndex;
result.workerIndex = this._workerIndex;
- result.shardIndex = this._config.config.shard?.current;
result.startTime = new Date(params.startWallTime);
this._reporter.onTestBegin?.(test, result);
this._currentlyRunning = { test, result };
diff --git a/packages/playwright/src/runner/testGroups.ts b/packages/playwright/src/runner/testGroups.ts
index c70ebd7586..7ebb2b5b31 100644
--- a/packages/playwright/src/runner/testGroups.ts
+++ b/packages/playwright/src/runner/testGroups.ts
@@ -133,7 +133,7 @@ export function createTestGroups(projectSuite: Suite, expectedParallelism: numbe
export function filterForShard(shard: { total: number, current: number }, weights: number[] | undefined, testGroups: TestGroup[]): Set<TestGroup> {
weights ??= Array.from({ length: shard.total }, () => 1);
if (weights.length !== shard.total)
- throw new Error(`--shard-weights number of weights must match the shard total of ${shard.total}`);
+ throw new Error(`PWTEST_SHARD_WEIGHTS number of weights must match the shard total of ${shard.total}`);
const totalWeight = weights.reduce((a, b) => a + b, 0);
// Note that sharding works based on test groups.
diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts
index 29292ea420..14154e7769 100644
--- a/packages/playwright/types/testReporter.d.ts
+++ b/packages/playwright/types/testReporter.d.ts
@@ -40,31 +40,6 @@ export interface FullResult {
* Test duration in milliseconds.
*/
duration: number;
-
- /**
- * Only present on merged reports.
- */
- shards?: {
- /**
- * The index of the shard, one-based.
- */
- shardIndex?: number;
-
- /**
- * Global tag that differentiates CI environments.
- */
- tag: string[];
-
- /**
- * Start wall time of shard.
- */
- startTime: Date;
-
- /**
- * Shard run duration in milliseconds.
- */
- duration: number;
- }[];
}
/**
@@ -706,11 +681,6 @@ export interface TestResult {
*/
retry: number;
- /**
- * The index of the shard between `1` and [`shards`](https://playwright.dev/docs/test-sharding).
- */
- shardIndex?: number;
-
/**
* Start time of this particular test run.
*/
diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts
index 40cab3922f..779087bbd7 100644
--- a/tests/playwright-test/playwright-test-fixtures.ts
+++ b/tests/playwright-test/playwright-test-fixtures.ts
@@ -237,6 +237,7 @@ export function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
PW_TEST_SOURCE_TRANSFORM: undefined,
PW_TEST_SOURCE_TRANSFORM_SCOPE: undefined,
PWTEST_BOT_NAME: undefined,
+ PWTEST_SHARD_WEIGHTS: undefined,
TEST_WORKER_INDEX: undefined,
TEST_PARALLEL_INDEX: undefined,
NODE_OPTIONS: undefined,
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index c3014f1102..9f4d299887 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -3344,20 +3344,14 @@ for (const useIntermediateMergeReport of [true, false] as const) {
await page.getByRole('link', { name: 'Speedboard' }).click();
await expect(page.getByRole('main')).toMatchAriaSnapshot(`
- - button "Shard Duration"
+ - button "Timeline"
- region:
- img:
- - list "@linux":
- - listitem /Shard 1/
- - listitem /Shard 2/
- - listitem /Shard 3/
- - list "@mac":
- - listitem /Shard 1/
- - listitem /Shard 2/
- - text: Use shard weights to
- - link "rebalance your shards":
- - /url: https://playwright.dev/docs/test-sharding#rebalancing-shards
- - text: /@linux. npx playwright test --shard-weights=\\d+:\\d+:\\d+ @mac. npx playwright test --shard-weights=\\d+:\\d+/
+ - listitem /@linux/
+ - listitem /@linux/
+ - listitem /@linux/
+ - listitem /@mac/
+ - listitem /@mac/
`);
});
});
diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts
index 18f4d52d12..89b995ff35 100644
--- a/tests/playwright-test/reporter-json.spec.ts
+++ b/tests/playwright-test/reporter-json.spec.ts
@@ -68,7 +68,6 @@ test('should not report skipped due to sharding', async ({ runInlineTest }) => {
expect(result.report.suites[0].specs.length).toBe(2);
expect(result.report.suites[0].specs[0].tests[0].status).toBe('expected');
expect(result.report.suites[0].specs[1].tests[0].status).toBe('skipped');
- expect(result.report.suites[0].specs[1].tests[0].results[0].shardIndex).toBe(1);
});
test('should report projects and stats', async ({ runInlineTest }, testInfo) => {
diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts
index bc3eb2f434..16ef26c251 100644
--- a/tests/playwright-test/shard.spec.ts
+++ b/tests/playwright-test/shard.spec.ts
@@ -335,7 +335,7 @@ test('should shard tests with beforeAll based on shards total instead of workers
test('should respect custom shard weights', async ({ runInlineTest }) => {
await test.step('shard 1', async () => {
- const result = await runInlineTest(tests, { 'shard': '1/2', 'shard-weights': '40:60', 'workers': 1 });
+ const result = await runInlineTest(tests, { 'shard': '1/2', 'workers': 1 }, { PWTEST_SHARD_WEIGHTS: '40:60' });
expect.soft(result.exitCode).toBe(0);
expect.soft(result.outputLines).toEqual([
'a1-test1-done',
@@ -345,7 +345,7 @@ test('should respect custom shard weights', async ({ runInlineTest }) => {
]);
});
await test.step('shard 2', async () => {
- const result = await runInlineTest(tests, { 'shard': '2/2', 'shard-weights': '40:60', 'workers': 1 });
+ const result = await runInlineTest(tests, { 'shard': '2/2', 'workers': 1 }, { PWTEST_SHARD_WEIGHTS: '40:60' });
expect.soft(result.exitCode).toBe(0);
expect.soft(result.outputLines).toEqual([
'a2-test1-done',
diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts
index 52fd354b04..5a32326fbc 100644
--- a/utils/generate_types/overrides-testReporter.d.ts
+++ b/utils/generate_types/overrides-testReporter.d.ts
@@ -39,31 +39,6 @@ export interface FullResult {
* Test duration in milliseconds.
*/
duration: number;
-
- /**
- * Only present on merged reports.
- */
- shards?: {
- /**
- * The index of the shard, one-based.
- */
- shardIndex?: number;
-
- /**
- * Global tag that differentiates CI environments.
- */
- tag: string[];
-
- /**
- * Start wall time of shard.
- */
- startTime: Date;
-
- /**
- * Shard run duration in milliseconds.
- */
- duration: number;
- }[];
}
export interface Reporter {