Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(html): add shard weight suggestion (#38696)

Simon Knott39190df338dc
diff --git a/docs/src/images/speedboard-shards.png b/docs/src/images/speedboard-shards.png
new file mode 100644
index 0000000000..3b39ccb2b1
Binary files /dev/null and b/docs/src/images/speedboard-shards.png differ
diff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.md
index 44b583f0f0..56190af17d 100644
--- a/docs/src/test-sharding-js.md
+++ b/docs/src/test-sharding-js.md
@@ -45,12 +45,21 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul
- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.
- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.
-## Rebalancing Shards
+### Rebalancing Shards
+
+If tests in one of your shards take longer, you can manually assign less work to it using the `--shard-weights` option:
```bash
-npx playwright test --shard=1/4 --shard-weights=26:24:25:25
+npx playwright test --shard=x/4 --shard-weights=3:2:3:3
```
+Make sure you pass the same `--shard-weights` value to all shards.
+In this example, Shard 2 gets assigned less tests because they take longer, evening out the full duration.
+
+In the Speedboard section of the combined HTML report, you can see a visualisation of your individual shard durations, and a recommendation for how to set your shard weights.
+
+![Speedboard Shard Diagram](./images/speedboard-shards.png)
+
## Merging reports from multiple shards
In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.
diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsx
index a4b52d501c..105450e94f 100644
--- a/packages/html-reporter/src/barchart.tsx
+++ b/packages/html-reporter/src/barchart.tsx
@@ -37,9 +37,8 @@ export const GroupedBarChart = ({
const width = 800;
// Calculate left margin based on longest group name
- // Rough estimate: 7 pixels per character at fontSize 12
const maxGroupNameLength = Math.max(...groups.map(g => g.length));
- const estimatedTextWidth = maxGroupNameLength * 7;
+ const estimatedTextWidth = maxGroupNameLength * 10;
const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));
const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };
@@ -78,7 +77,7 @@ export const GroupedBarChart = ({
// Allocate space proportionally based on number of bars
const barHeight = 20; // Fixed bar height
const barSpacing = 4;
- const groupPadding = 12;
+ const groupPadding = 20;
// Calculate Y positions for each group based on their bar count
const groupYPositions: number[] = [];
@@ -149,7 +148,7 @@ export const GroupedBarChart = ({
const y = groupY + barIndex * (barHeight + barSpacing);
barIndex++;
- const colors = ['var(--color-scale-yellow-3)', 'var(--color-scale-orange-4)', 'var(--color-scale-blue-3)', 'var(--color-scale-green-3)'];
+ const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];
const color = colors[seriesIndex % colors.length];
return (
diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsx
index b461a2272e..4f8d84ce4c 100644
--- a/packages/html-reporter/src/speedboard.tsx
+++ b/packages/html-reporter/src/speedboard.tsx
@@ -21,6 +21,7 @@ import * as icons from './icons';
import { TestCaseSummary } from './types';
import { AutoChip } from './chip';
import { GroupedBarChart } from './barchart';
+import { CodeSnippet } from './testErrorView';
export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {
return <>
@@ -56,21 +57,23 @@ export function Shards({ report }: { report: LoadedReport }) {
return null;
let clash = false;
- const bots: Record<string, number[]> = {};
+ const bots: Record<string, { durations: number[], weights: number[] }> = {};
for (const shard of shards) {
const botName = shard.tag.join(' ');
- bots[botName] ??= [];
+ bots[botName] ??= { durations: [], weights: [] };
const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0);
- if (bots[botName][shardIndex] !== undefined)
+ if (bots[botName].durations[shardIndex] !== undefined)
clash = true;
- bots[botName][shardIndex] = shard.duration;
+ bots[botName].durations[shardIndex] = shard.duration;
+ bots[botName].weights[shardIndex] = shard.suggestedWeight;
}
- const maxSeries = Math.max(...Object.values(bots).map(shardDurations => shardDurations.length));
+ const maxSeries = Math.max(...Object.values(bots).map(b => b.durations.length));
+ const weightsSnippet = formatWeightCommands(bots);
return <AutoChip header='Shard Duration'>
<GroupedBarChart
- data={Object.values(bots)}
+ data={Object.values(bots).map(b => b.durations)}
groups={Object.keys(bots)}
series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}
/>
@@ -81,5 +84,25 @@ export function Shards({ report }: { report: LoadedReport }) {
the docs
</a> on how to fix this.
</div>}
+ {weightsSnippet && <>
+ <div style={{ marginTop: 8, marginBottom: 16 }}>
+ Use shard weights to <a href='https://playwright.dev/docs/test-sharding#rebalancing-shards' target='_blank' rel='noopener noreferrer'>
+ rebalance your shards
+ </a>:
+ </div>
+ <CodeSnippet code={weightsSnippet} />
+ </>}
</AutoChip>;
}
+
+function formatWeightCommands(bots: Record<string, { weights: number[] }>): string | undefined {
+ const entries = Object.entries(bots).filter(([, { weights }]) => weights.length > 1);
+ if (entries.length === 0)
+ return;
+
+ const maxLen = Math.max(...entries.map(([botName]) => botName.length));
+ return entries.map(([botName, { weights }]) => {
+ const prefix = botName ? `${botName}:`.padEnd(maxLen + 2) : '';
+ return `${prefix}npx playwright test \x1b[33m--shard-weights=\x1b[35m${weights.join(':')}\x1b[39m`;
+ }).join('\n');
+}
diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts
index ce913a55a9..6f288bff80 100644
--- a/packages/html-reporter/src/types.d.ts
+++ b/packages/html-reporter/src/types.d.ts
@@ -54,6 +54,7 @@ export type HTMLReport = {
tag: string[];
startTime: number;
duration: number;
+ suggestedWeight: number;
}[];
errors: string[]; // Top-level errors that are not attributed to any test.
options: HTMLReportOptions;
diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts
index ad1942a88a..69564b6dc6 100644
--- a/packages/playwright/src/reporters/html.ts
+++ b/packages/playwright/src/reporters/html.ts
@@ -298,12 +298,7 @@ class HtmlBuilder {
stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },
errors: topLevelErrors.map(error => formatError(internalScreen, error).message),
options: this._options,
- shards: result.shards?.map(s => ({
- shardIndex: s.shardIndex,
- tag: s.tag,
- startTime: s.startTime.getTime(),
- duration: s.duration,
- })) ?? []
+ shards: this._buildShardsInfo(result, projectSuites),
};
htmlReport.files.sort((f1, f2) => {
const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;
@@ -347,6 +342,99 @@ class HtmlBuilder {
return { ok, singleTestId };
}
+ private _buildShardsInfo(result: api.FullResult, projectSuites: api.Suite[]): HTMLReport['shards'] {
+ if (!result.shards)
+ return [];
+
+ const shardsInfo: HTMLReport['shards'] = result.shards.map(s => ({
+ duration: s.duration,
+ startTime: s.startTime.getTime(),
+ tag: s.tag,
+ shardIndex: s.shardIndex,
+ suggestedWeight: 100,
+ }));
+
+ const bots: Record<string, api.TestCase[]> = {};
+
+ for (const projectSuite of projectSuites) {
+ for (const test of projectSuite.allTests()) {
+ let longestPrefix: string[] = [];
+ for (const { tag } of shardsInfo) {
+ if (longestPrefix.length < tag.length && tag.every((v, i) => test.tags[i] === v))
+ longestPrefix = tag;
+ }
+ const botName = longestPrefix.join(';');
+ bots[botName] ??= [];
+ bots[botName].push(test);
+ }
+ }
+
+ for (const [botName, tests] of Object.entries(bots)) {
+ const botShards = shardsInfo.filter(shard => shard.tag.join(';') === botName);
+ const totalShard = Math.max(...botShards.map(s => s.shardIndex ?? -1));
+ if (totalShard < 2)
+ continue;
+ let suggestedWeights = this._calculateShardWeights(tests, totalShard);
+ suggestedWeights = this._normaliseWeights(suggestedWeights);
+ for (const [index, weight] of suggestedWeights.entries()) {
+ const shard = botShards.find(s => s.shardIndex === index + 1);
+ if (shard)
+ shard.suggestedWeight = weight;
+ }
+ }
+
+ return shardsInfo;
+ }
+
+ private _calculateShardWeights(tests: api.TestCase[], shardTotal: number): number[] {
+ if (tests.length === 0)
+ return [];
+
+ // recreates the sorting that filterForShard gets as input
+ tests.sort((a, b) => {
+ const shardIndexA = a.results[0]!.shardIndex ?? -1;
+ const shardIndexB = b.results[0]!.shardIndex ?? -1;
+ if (shardIndexA !== shardIndexB)
+ return shardIndexA - shardIndexB;
+
+ const startTimeA = a.results[0]!.startTime.getTime();
+ const startTimeB = b.results[0]!.startTime.getTime();
+ return startTimeA - startTimeB;
+ });
+
+ const totalDuration = tests.map(test => test.results[0]!.duration).reduce((a, b) => a + b, 0);
+ const optimalDuration = totalDuration / shardTotal;
+
+ const weights: number[] = [];
+ let currentDuration = 0;
+ let currentTestCount = 0;
+
+ for (const test of tests) {
+ const duration = test.results[0]!.duration;
+ currentDuration += duration;
+ currentTestCount++;
+
+ // When we've reached the optimal shard duration, record this shard's weight
+ if (currentDuration >= optimalDuration && weights.length < shardTotal - 1) {
+ weights.push(currentTestCount);
+ currentDuration = 0;
+ currentTestCount = 0;
+ }
+ }
+ weights.push(currentTestCount);
+
+ return weights;
+ }
+
+ private _normaliseWeights(weights: number[]): number[] {
+ const total = weights.reduce((a, b) => a + b, 0);
+ weights = weights.map(w => Math.floor((w / total) * 100));
+ const remaining = 100 - weights.reduce((a, b) => a + b, 0);
+ for (let i = 0; i < remaining; i++)
+ weights[i % weights.length]++;
+ return weights;
+ }
+
private async _writeReportData(filePath: string) {
fs.appendFileSync(filePath, '<script id="playwrightReportBase64" type="application/zip">data:application/zip;base64,');
await new Promise(f => {
diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index e0d502b797..128e3c417e 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -3354,6 +3354,10 @@ for (const useIntermediateMergeReport of [true, false] as const) {
- list "@mac":
- listitem /Shard 1/
- listitem /Shard 2/
+ - text: Use shard weights to
+ - link "rebalance your shards":
+ - /url: https://playwright.dev/docs/test-sharding#rebalancing-shards
+ - text: ": @linux: npx playwright test --shard-weights=67:33:0 @mac: npx playwright test --shard-weights=67:33"
`);
});
});