Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore(html): add shard weight suggestion (#38696)
Simon Knott39190df338dc
diff --git a/docs/src/images/speedboard-shards.png b/docs/src/images/speedboard-shards.pngnew file mode 100644index 0000000000..3b39ccb2b1Binary files /dev/null and b/docs/src/images/speedboard-shards.png differdiff --git a/docs/src/test-sharding-js.md b/docs/src/test-sharding-js.mdindex 44b583f0f0..56190af17d 100644--- a/docs/src/test-sharding-js.md+++ b/docs/src/test-sharding-js.md@@ -45,12 +45,21 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul- **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.- To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.-## Rebalancing Shards+### Rebalancing Shards++If tests in one of your shards take longer, you can manually assign less work to it using the `--shard-weights` option:```bash-npx playwright test --shard=1/4 --shard-weights=26:24:25:25+npx playwright test --shard=x/4 --shard-weights=3:2:3:3```+Make sure you pass the same `--shard-weights` value to all shards.+In this example, Shard 2 gets assigned less tests because they take longer, evening out the full duration.++In the Speedboard section of the combined HTML report, you can see a visualisation of your individual shard durations, and a recommendation for how to set your shard weights.+++## Merging reports from multiple shardsIn the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsxindex a4b52d501c..105450e94f 100644--- a/packages/html-reporter/src/barchart.tsx+++ b/packages/html-reporter/src/barchart.tsx@@ -37,9 +37,8 @@ export const GroupedBarChart = ({const width = 800;// Calculate left margin based on longest group name- // Rough estimate: 7 pixels per character at fontSize 12const maxGroupNameLength = Math.max(...groups.map(g => g.length));- const estimatedTextWidth = maxGroupNameLength * 7;+ const estimatedTextWidth = maxGroupNameLength * 10;const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth));const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };@@ -78,7 +77,7 @@ export const GroupedBarChart = ({// Allocate space proportionally based on number of barsconst barHeight = 20; // Fixed bar heightconst barSpacing = 4;- const groupPadding = 12;+ const groupPadding = 20;// Calculate Y positions for each group based on their bar countconst groupYPositions: number[] = [];@@ -149,7 +148,7 @@ export const GroupedBarChart = ({const y = groupY + barIndex * (barHeight + barSpacing);barIndex++;- const colors = ['var(--color-scale-yellow-3)', 'var(--color-scale-orange-4)', 'var(--color-scale-blue-3)', 'var(--color-scale-green-3)'];+ const colors = ['var(--color-scale-blue-2)', 'var(--color-scale-blue-3)', 'var(--color-scale-blue-4)'];const color = colors[seriesIndex % colors.length];return (diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsxindex b461a2272e..4f8d84ce4c 100644--- a/packages/html-reporter/src/speedboard.tsx+++ b/packages/html-reporter/src/speedboard.tsx@@ -21,6 +21,7 @@ import * as icons from './icons';import { TestCaseSummary } from './types';import { AutoChip } from './chip';import { GroupedBarChart } from './barchart';+import { CodeSnippet } from './testErrorView';export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {return <>@@ -56,21 +57,23 @@ export function Shards({ report }: { report: LoadedReport }) {return null;let clash = false;- const bots: Record<string, number[]> = {};+ const bots: Record<string, { durations: number[], weights: number[] }> = {};for (const shard of shards) {const botName = shard.tag.join(' ');- bots[botName] ??= [];+ bots[botName] ??= { durations: [], weights: [] };const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0);- if (bots[botName][shardIndex] !== undefined)+ if (bots[botName].durations[shardIndex] !== undefined)clash = true;- bots[botName][shardIndex] = shard.duration;+ bots[botName].durations[shardIndex] = shard.duration;+ bots[botName].weights[shardIndex] = shard.suggestedWeight;}- const maxSeries = Math.max(...Object.values(bots).map(shardDurations => shardDurations.length));+ const maxSeries = Math.max(...Object.values(bots).map(b => b.durations.length));+ const weightsSnippet = formatWeightCommands(bots);return <AutoChip header='Shard Duration'><GroupedBarChart- data={Object.values(bots)}+ data={Object.values(bots).map(b => b.durations)}groups={Object.keys(bots)}series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}/>@@ -81,5 +84,25 @@ export function Shards({ report }: { report: LoadedReport }) {the docs</a> on how to fix this.</div>}+ {weightsSnippet && <>+ <div style={{ marginTop: 8, marginBottom: 16 }}>+ Use shard weights to <a href='https://playwright.dev/docs/test-sharding#rebalancing-shards' target='_blank' rel='noopener noreferrer'>+ rebalance your shards+ </a>:+ </div>+ <CodeSnippet code={weightsSnippet} />+ </>}</AutoChip>;}++function formatWeightCommands(bots: Record<string, { weights: number[] }>): string | undefined {+ const entries = Object.entries(bots).filter(([, { weights }]) => weights.length > 1);+ if (entries.length === 0)+ return;++ const maxLen = Math.max(...entries.map(([botName]) => botName.length));+ return entries.map(([botName, { weights }]) => {+ const prefix = botName ? `${botName}:`.padEnd(maxLen + 2) : '';+ return `${prefix}npx playwright test \x1b[33m--shard-weights=\x1b[35m${weights.join(':')}\x1b[39m`;+ }).join('\n');+}diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.tsindex ce913a55a9..6f288bff80 100644--- a/packages/html-reporter/src/types.d.ts+++ b/packages/html-reporter/src/types.d.ts@@ -54,6 +54,7 @@ export type HTMLReport = {tag: string[];startTime: number;duration: number;+ suggestedWeight: number;}[];errors: string[]; // Top-level errors that are not attributed to any test.options: HTMLReportOptions;diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.tsindex ad1942a88a..69564b6dc6 100644--- a/packages/playwright/src/reporters/html.ts+++ b/packages/playwright/src/reporters/html.ts@@ -298,12 +298,7 @@ class HtmlBuilder {stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },errors: topLevelErrors.map(error => formatError(internalScreen, error).message),options: this._options,- shards: result.shards?.map(s => ({- shardIndex: s.shardIndex,- tag: s.tag,- startTime: s.startTime.getTime(),- duration: s.duration,- })) ?? []+ shards: this._buildShardsInfo(result, projectSuites),};htmlReport.files.sort((f1, f2) => {const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;@@ -347,6 +342,99 @@ class HtmlBuilder {return { ok, singleTestId };}+ private _buildShardsInfo(result: api.FullResult, projectSuites: api.Suite[]): HTMLReport['shards'] {+ if (!result.shards)+ return [];++ const shardsInfo: HTMLReport['shards'] = result.shards.map(s => ({+ duration: s.duration,+ startTime: s.startTime.getTime(),+ tag: s.tag,+ shardIndex: s.shardIndex,+ suggestedWeight: 100,+ }));++ const bots: Record<string, api.TestCase[]> = {};++ for (const projectSuite of projectSuites) {+ for (const test of projectSuite.allTests()) {+ let longestPrefix: string[] = [];+ for (const { tag } of shardsInfo) {+ if (longestPrefix.length < tag.length && tag.every((v, i) => test.tags[i] === v))+ longestPrefix = tag;+ }+ const botName = longestPrefix.join(';');+ bots[botName] ??= [];+ bots[botName].push(test);+ }+ }++ for (const [botName, tests] of Object.entries(bots)) {+ const botShards = shardsInfo.filter(shard => shard.tag.join(';') === botName);+ const totalShard = Math.max(...botShards.map(s => s.shardIndex ?? -1));+ if (totalShard < 2)+ continue;+ let suggestedWeights = this._calculateShardWeights(tests, totalShard);+ suggestedWeights = this._normaliseWeights(suggestedWeights);+ for (const [index, weight] of suggestedWeights.entries()) {+ const shard = botShards.find(s => s.shardIndex === index + 1);+ if (shard)+ shard.suggestedWeight = weight;+ }+ }++ return shardsInfo;+ }++ private _calculateShardWeights(tests: api.TestCase[], shardTotal: number): number[] {+ if (tests.length === 0)+ return [];++ // recreates the sorting that filterForShard gets as input+ tests.sort((a, b) => {+ const shardIndexA = a.results[0]!.shardIndex ?? -1;+ const shardIndexB = b.results[0]!.shardIndex ?? -1;+ if (shardIndexA !== shardIndexB)+ return shardIndexA - shardIndexB;++ const startTimeA = a.results[0]!.startTime.getTime();+ const startTimeB = b.results[0]!.startTime.getTime();+ return startTimeA - startTimeB;+ });++ const totalDuration = tests.map(test => test.results[0]!.duration).reduce((a, b) => a + b, 0);+ const optimalDuration = totalDuration / shardTotal;++ const weights: number[] = [];+ let currentDuration = 0;+ let currentTestCount = 0;++ for (const test of tests) {+ const duration = test.results[0]!.duration;+ currentDuration += duration;+ currentTestCount++;++ // When we've reached the optimal shard duration, record this shard's weight+ if (currentDuration >= optimalDuration && weights.length < shardTotal - 1) {+ weights.push(currentTestCount);+ currentDuration = 0;+ currentTestCount = 0;+ }+ }+ weights.push(currentTestCount);++ return weights;+ }++ private _normaliseWeights(weights: number[]): number[] {+ const total = weights.reduce((a, b) => a + b, 0);+ weights = weights.map(w => Math.floor((w / total) * 100));+ const remaining = 100 - weights.reduce((a, b) => a + b, 0);+ for (let i = 0; i < remaining; i++)+ weights[i % weights.length]++;+ return weights;+ }+private async _writeReportData(filePath: string) {fs.appendFileSync(filePath, '<script id="playwrightReportBase64" type="application/zip">data:application/zip;base64,');await new Promise(f => {diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.tsindex e0d502b797..128e3c417e 100644--- a/tests/playwright-test/reporter-html.spec.ts+++ b/tests/playwright-test/reporter-html.spec.ts@@ -3354,6 +3354,10 @@ for (const useIntermediateMergeReport of [true, false] as const) {- list "@mac":- listitem /Shard 1/- listitem /Shard 2/+ - text: Use shard weights to+ - link "rebalance your shards":+ - /url: https://playwright.dev/docs/test-sharding#rebalancing-shards+ - text: ": @linux: npx playwright test --shard-weights=67:33:0 @mac: npx playwright test --shard-weights=67:33"`);});});