Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

⏱️ ci: Balance Playwright Shards by Recorded Duration (#17019)

Danny Avila27e8930d5205
diff --git a/.github/scripts/playwright-matrix.test.mjs b/.github/scripts/playwright-matrix.test.mjs
index 2a5d578ff8..ab7bc27e1d 100644
--- a/.github/scripts/playwright-matrix.test.mjs
+++ b/.github/scripts/playwright-matrix.test.mjs
@@ -85,19 +85,19 @@ function decision(redis, mcp = true) {
return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } };
}
-test('PR selection and fail-open defaults use six memory shards and two Redis transport shards', () => {
+test('PR selection and fail-open defaults use ten memory shards and two Redis transport shards', () => {
assert.equal(defaults.length, 2);
assert.deepEqual(defaults[0], prMatrix);
- assert.equal(prMatrix.include.length, 8);
- assertPartition(prMatrix, 'memory', 'full', 6);
+ assert.equal(prMatrix.include.length, 12);
+ assertPartition(prMatrix, 'memory', 'full', 10);
assertPartition(prMatrix, 'redis', 'transport', 2);
});
test('nightly and default manual runs retain full-suite coverage on both stores', () => {
assert.equal(workflow.on.workflow_dispatch.inputs.coverage.default, 'full');
- assert.equal(defaults[1].include.length, 4);
- assertPartition(defaults[1], 'memory', 'full', 2);
- assertPartition(defaults[1], 'redis', 'full', 2);
+ assert.equal(defaults[1].include.length, 20);
+ assertPartition(defaults[1], 'memory', 'full', 10);
+ assertPartition(defaults[1], 'redis', 'full', 10);
});
test('PR and nightly matrices preserve both registered-email change modes on one memory shard', () => {
@@ -107,7 +107,10 @@ test('PR and nightly matrices preserve both registered-email change modes on one
assert.equal(emailLanes[0].stream_store, 'memory');
assert.match(emailLanes[0].shard, /^1\//);
}
- assert.equal(shards.steps.filter((step) => step.if === "matrix.email_change == 'true'").length, 2);
+ assert.equal(
+ shards.steps.filter((step) => step.if === "matrix.email_change == 'true'").length,
+ 2,
+ );
});
test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => {
@@ -129,9 +132,12 @@ test('every shard has a unique job name and artifact namespace', () => {
});
test('both suite commands consume the matrix shard without increasing worker concurrency', () => {
- for (const suite of ['full', 'transport']) {
- const step = shards.steps.find((entry) => entry.if === `matrix.suite == '${suite}'`);
- assert.match(step.run, /--shard=\$\{\{ matrix\.shard \}\}/);
+ const full = shards.steps.find((entry) => entry.if === "matrix.suite == 'full'");
+ assert.match(full.run, /playwright-shards\.mjs plan[\s\S]*--shard '\$\{\{ matrix\.shard \}\}'/);
+ assert.match(full.run, /--test-list=e2e\/\.generated\/shard-tests\.txt/);
+ const transport = shards.steps.find((entry) => entry.if === "matrix.suite == 'transport'");
+ assert.match(transport.run, /--shard=\$\{\{ matrix\.shard \}\}/);
+ for (const step of [full, transport]) {
assert.doesNotMatch(step.run, /--workers/);
}
const config = readFileSync(
@@ -146,8 +152,8 @@ test('an explicit Redis skip drops both transport shards and no memory shard', (
const { matrix, outputs } = select(t, decision(false, false));
assert.equal(outputs.decided, 'true');
assert.equal(outputs.mcp_run, 'false');
- assert.equal(matrix.include.length, 6);
- assertPartition(matrix, 'memory', 'full', 6);
+ assert.equal(matrix.include.length, 10);
+ assertPartition(matrix, 'memory', 'full', 10);
});
test('missing or non-boolean Redis decisions retain both transport shards', (t) => {
@@ -198,7 +204,12 @@ test('graduated spec skips keep the same pool across shards and remain opt-in',
test('the required gate still waits for all shards and rejects failures without fighting cancellation', () => {
const gate = workflow.jobs.e2e;
assert.equal(gate.name, 'e2e');
- assert.deepEqual(gate.needs, ['codegraph_select', 'e2e_shards', 'mcp_tool_list_changed', 'mcp_apps']);
+ assert.deepEqual(gate.needs, [
+ 'codegraph_select',
+ 'e2e_shards',
+ 'mcp_tool_list_changed',
+ 'mcp_apps',
+ ]);
assert.equal(shards.strategy['fail-fast'], false);
assert.match(gate.if, /!cancelled\(\)/);
assert.match(shards.if, /!cancelled\(\)/);
diff --git a/.github/scripts/playwright-shards.mjs b/.github/scripts/playwright-shards.mjs
new file mode 100644
index 0000000000..8ba7e99bc5
--- /dev/null
+++ b/.github/scripts/playwright-shards.mjs
@@ -0,0 +1,265 @@
+#!/usr/bin/env node
+/**
+ * Duration-balanced Playwright shards.
+ *
+ * `--shard=i/n` cuts the ordered test list into n runs of equal *count*, so a shard that
+ * happens to draw the slow, streaming-heavy specs runs minutes longer than one that draws
+ * quick visual checks, and the run waits for the slowest. This assigns whole spec files
+ * instead, longest first, each to the shard with the least estimated work so far (LPT), using
+ * the per-file seconds recorded in the timings file. A file with no recorded time is
+ * estimated from its test count at the recorded average per test, so new specs still spread.
+ *
+ * Every shard computes the same partition from the same inputs (Playwright's own `--list` of
+ * the config, the timings file and the reservations), and the shards are disjoint and cover
+ * every listed file, so the union is exactly the suite `--shard` would have run.
+ *
+ * plan --list <list.json> --timings <timings.json> --shard <i/n>
+ * [--reserve <i:seconds>[,<i:seconds>...]] [--skip "<specs/mock/a.spec.ts> ..."]
+ * --out <test-list.txt>
+ * Writes the shard's files as a Playwright `--test-list` and prints the plan.
+ * `--reserve` charges a shard for work it does outside this run (extra steps), so the
+ * partition gives it less. `--skip` drops files (paths relative to e2e/) before
+ * partitioning; a skip that would drop every file is ignored.
+ *
+ * record --out <timings.json> <dir-or-results.json>...
+ * Rebuilds the timings file from Playwright JSON reports (`results.json`, written beside
+ * each shard's HTML report in CI). Download a run's `playwright-report-memory-*`
+ * artifacts and point this at them.
+ */
+import { readFileSync, readdirSync, statSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+/** Paths in a skip list are relative to `e2e/`; Playwright lists them relative to the testDir. */
+const SKIP_PREFIX = 'specs/mock/';
+
+function parseArgs(argv) {
+ const [command, ...rest] = argv;
+ const options = { _: [] };
+ for (let i = 0; i < rest.length; i++) {
+ const arg = rest[i];
+ if (!arg.startsWith('--')) {
+ options._.push(arg);
+ continue;
+ }
+ const eq = arg.indexOf('=');
+ if (eq !== -1) {
+ options[arg.slice(2, eq)] = arg.slice(eq + 1);
+ continue;
+ }
+ options[arg.slice(2)] = rest[++i];
+ }
+ return { command, options };
+}
+
+function parseShard(value) {
+ const match = /^(\d+)\/(\d+)$/.exec(value ?? '');
+ const index = match ? Number(match[1]) : NaN;
+ const total = match ? Number(match[2]) : NaN;
+ if (!(total >= 1 && index >= 1 && index <= total)) {
+ throw new Error(`--shard must look like i/n with 1 <= i <= n; got ${value}`);
+ }
+ return { index, total };
+}
+
+function parseReserve(value, total) {
+ const reserve = new Array(total).fill(0);
+ for (const entry of (value ?? '').split(',').filter(Boolean)) {
+ const match = /^(\d+):(\d+(?:\.\d+)?)$/.exec(entry.trim());
+ if (!match || Number(match[1]) < 1 || Number(match[1]) > total) {
+ throw new Error(
+ `--reserve entries must look like i:seconds with 1 <= i <= ${total}; got ${entry}`,
+ );
+ }
+ reserve[Number(match[1]) - 1] += Number(match[2]);
+ }
+ return reserve;
+}
+
+/** Test count per file from `playwright test --list --reporter=json`, in listing order. */
+export function listedFiles(report) {
+ if (report.errors?.length) {
+ const first = report.errors[0];
+ throw new Error(
+ `playwright --list reported ${report.errors.length} error(s): ${first.message ?? first}`,
+ );
+ }
+ const counts = new Map();
+ const walk = (suite, file) => {
+ for (const spec of suite.specs ?? []) {
+ counts.set(file, (counts.get(file) ?? 0) + spec.tests.length);
+ }
+ for (const child of suite.suites ?? []) {
+ walk(child, file);
+ }
+ };
+ for (const suite of report.suites ?? []) {
+ counts.set(suite.file, counts.get(suite.file) ?? 0);
+ walk(suite, suite.file);
+ }
+ for (const file of counts.keys()) {
+ if (/[>›#]/.test(file) || file !== file.trim()) {
+ throw new Error(`cannot express ${JSON.stringify(file)} in a Playwright test list`);
+ }
+ }
+ return counts;
+}
+
+/** Seconds for each file: the recorded time, or its test count at the recorded mean. */
+export function estimate(counts, timings) {
+ let recordedSeconds = 0;
+ let recordedTests = 0;
+ for (const [file, tests] of counts) {
+ if (typeof timings[file] === 'number' && tests > 0) {
+ recordedSeconds += timings[file];
+ recordedTests += tests;
+ }
+ }
+ const perTest = recordedTests > 0 ? recordedSeconds / recordedTests : 1;
+ return new Map(
+ [...counts].map(([file, tests]) => [
+ file,
+ typeof timings[file] === 'number' ? timings[file] : tests * perTest,
+ ]),
+ );
+}
+
+/**
+ * Longest-processing-time-first: files in descending estimate (ties by name), each to the
+ * least-loaded shard (ties to the lower index). Deterministic for identical inputs.
+ */
+export function partition(estimates, total, reserve = new Array(total).fill(0)) {
+ const loads = [...reserve];
+ const shards = Array.from({ length: total }, () => []);
+ const ordered = [...estimates].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
+ for (const [file, seconds] of ordered) {
+ let target = 0;
+ for (let i = 1; i < total; i++) {
+ if (loads[i] < loads[target]) {
+ target = i;
+ }
+ }
+ shards[target].push(file);
+ loads[target] += seconds;
+ }
+ for (const files of shards) {
+ files.sort();
+ }
+ return { shards, loads };
+}
+
+export function applySkip(counts, skip) {
+ const names = new Set(
+ (skip ?? '')
+ .split(/\s+/)
+ .filter(Boolean)
+ .map((path) => (path.startsWith(SKIP_PREFIX) ? path.slice(SKIP_PREFIX.length) : path)),
+ );
+ if (names.size === 0) {
+ return { counts, dropped: 0 };
+ }
+ const kept = new Map([...counts].filter(([file]) => !names.has(file)));
+ const dropped = counts.size - kept.size;
+ if (dropped === 0 || kept.size === 0) {
+ return { counts, dropped: 0 };
+ }
+ return { counts: kept, dropped };
+}
+
+function plan(options) {
+ const { index, total } = parseShard(options.shard);
+ if (!options.list || !options.timings || !options.out) {
+ throw new Error('plan needs --list, --timings and --out');
+ }
+ const listed = listedFiles(JSON.parse(readFileSync(options.list, 'utf8')));
+ if (listed.size === 0) {
+ throw new Error('playwright --list found no spec files');
+ }
+ const timings = JSON.parse(readFileSync(options.timings, 'utf8')).files ?? {};
+ const { counts, dropped } = applySkip(listed, options.skip);
+ const estimates = estimate(counts, timings);
+ const { shards, loads } = partition(estimates, total, parseReserve(options.reserve, total));
+ const mine = shards[index - 1];
+ mkdirSync(dirname(options.out), { recursive: true });
+ writeFileSync(options.out, mine.map((file) => `${file}\n`).join(''));
+ const unrecorded = [...counts.keys()].filter((file) => typeof timings[file] !== 'number');
+ if (dropped) {
+ console.log(`codegraph-e2e-skip: dropped ${dropped} graduated specs from this shard's pool`);
+ }
+ console.log(
+ `playwright-shards: shard ${index}/${total} runs ${mine.length} of ${counts.size} files ` +
+ `(~${Math.round(loads[index - 1])}s estimated)`,
+ );
+ if (mine.length === 0) {
+ console.log('::warning::this shard drew no spec files; the suite has fewer files than shards');
+ }
+ console.log(`playwright-shards: estimated seconds per shard: ${loads.map(Math.round).join(' ')}`);
+ if (unrecorded.length) {
+ console.log(
+ `playwright-shards: ${unrecorded.length} file(s) have no recorded time; estimated from test counts`,
+ );
+ }
+}
+
+function findReports(path) {
+ if (statSync(path).isDirectory()) {
+ return readdirSync(path).flatMap((entry) => findReports(join(path, entry)));
+ }
+ return path.endsWith('.json') && path.split(/[\\/]/).pop() === 'results.json' ? [path] : [];
+}
+
+/** Sum of every attempt's duration per file: what a shard actually spends on it. */
+export function fileSeconds(report) {
+ const seconds = new Map();
+ const walk = (suite, file) => {
+ for (const spec of suite.specs ?? []) {
+ for (const test of spec.tests) {
+ for (const result of test.results ?? []) {
+ seconds.set(file, (seconds.get(file) ?? 0) + (result.duration ?? 0) / 1000);
+ }
+ }
+ }
+ for (const child of suite.suites ?? []) {
+ walk(child, file);
+ }
+ };
+ for (const suite of report.suites ?? []) {
+ walk(suite, suite.file);
+ }
+ return seconds;
+}
+
+function record(options) {
+ if (!options.out || options._.length === 0) {
+ throw new Error('record needs --out and at least one report path');
+ }
+ const totals = new Map();
+ const reports = options._.flatMap(findReports);
+ for (const path of reports) {
+ for (const [file, seconds] of fileSeconds(JSON.parse(readFileSync(path, 'utf8')))) {
+ totals.set(file, (totals.get(file) ?? 0) + seconds);
+ }
+ }
+ if (totals.size === 0) {
+ throw new Error(`no test results found in ${reports.length} report(s)`);
+ }
+ const files = Object.fromEntries(
+ [...totals]
+ .sort(([a], [b]) => (a < b ? -1 : 1))
+ .map(([file, s]) => [file, Math.round(s * 10) / 10]),
+ );
+ writeFileSync(options.out, `${JSON.stringify({ files }, null, 2)}\n`);
+ console.log(`playwright-shards: recorded ${totals.size} files from ${reports.length} report(s)`);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
+ const { command, options } = parseArgs(process.argv.slice(2));
+ if (command === 'plan') {
+ plan(options);
+ } else if (command === 'record') {
+ record(options);
+ } else {
+ console.error('usage: playwright-shards.mjs plan|record ...');
+ process.exit(2);
+ }
+}
diff --git a/.github/scripts/playwright-shards.test.mjs b/.github/scripts/playwright-shards.test.mjs
new file mode 100644
index 0000000000..d28754aa48
--- /dev/null
+++ b/.github/scripts/playwright-shards.test.mjs
@@ -0,0 +1,263 @@
+import { test } from 'node:test';
+import { join } from 'node:path';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
+import { applySkip, estimate, fileSeconds, listedFiles, partition } from './playwright-shards.mjs';
+
+const script = fileURLToPath(new URL('./playwright-shards.mjs', import.meta.url));
+const scriptsDir = fileURLToPath(new URL('.', import.meta.url));
+
+/** A `--list --reporter=json` report: one top-level suite per file, tests nested in describes. */
+function listing(files) {
+ return {
+ errors: [],
+ suites: Object.entries(files).map(([file, tests]) => ({
+ file,
+ specs: [],
+ suites: [
+ {
+ file,
+ specs: Array.from({ length: tests }, (_, i) => ({ title: `t${i}`, tests: [{}] })),
+ suites: [],
+ },
+ ],
+ })),
+ };
+}
+
+function scratch(t) {
+ const dir = mkdtempSync(join(scriptsDir, '.playwright-shards-test-'));
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
+ return dir;
+}
+
+function plan(t, { files, timings, shard, reserve = '', skip = '' }) {
+ const dir = scratch(t);
+ writeFileSync(join(dir, 'list.json'), JSON.stringify(listing(files)));
+ writeFileSync(join(dir, 'timings.json'), JSON.stringify({ files: timings }));
+ const out = join(dir, 'out', 'shard.txt');
+ const result = spawnSync(
+ process.execPath,
+ [
+ script,
+ 'plan',
+ '--list',
+ join(dir, 'list.json'),
+ '--timings',
+ join(dir, 'timings.json'),
+ '--shard',
+ shard,
+ '--reserve',
+ reserve,
+ '--skip',
+ skip,
+ '--out',
+ out,
+ ],
+ { encoding: 'utf8' },
+ );
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ return {
+ files: readFileSync(out, 'utf8').split('\n').filter(Boolean),
+ stdout: result.stdout,
+ };
+}
+
+const suite = {
+ 'a.spec.ts': 3,
+ 'b.spec.ts': 1,
+ 'c.spec.ts': 2,
+ 'scenarios/d.spec.ts': 4,
+ 'scenarios/e.spec.ts': 1,
+ 'f.spec.ts': 5,
+ 'g.spec.ts': 2,
+};
+const recorded = {
+ 'a.spec.ts': 30,
+ 'b.spec.ts': 120,
+ 'c.spec.ts': 10,
+ 'scenarios/d.spec.ts': 40,
+ 'f.spec.ts': 50,
+ 'g.spec.ts': 15,
+ 'removed.spec.ts': 999,
+};
+
+test('every listed file runs in exactly one shard, whatever the shard count', (t) => {
+ for (const total of [1, 2, 3, 6, 10]) {
+ const seen = [];
+ for (let index = 1; index <= total; index++) {
+ seen.push(...plan(t, { files: suite, timings: recorded, shard: `${index}/${total}` }).files);
+ }
+ assert.deepEqual(seen.sort(), Object.keys(suite).sort(), `total ${total}`);
+ }
+});
+
+test('shards balance recorded seconds rather than test counts', () => {
+ const counts = listedFiles(listing(suite));
+ const { loads, shards } = partition(estimate(counts, recorded), 2);
+ /** The 120s single-test file sits with at most one small file: halves by test count would
+ * have paired it with four others and doubled that shard. */
+ assert.ok(shards[0].includes('b.spec.ts') && shards[0].length <= 2, `${shards[0]}`);
+ assert.ok(Math.max(...loads) - Math.min(...loads) <= 30, `loads ${loads}`);
+});
+
+test('an unrecorded file is estimated from its test count at the recorded mean per test', () => {
+ const counts = listedFiles(listing(suite));
+ const estimates = estimate(counts, recorded);
+ const known = [
+ 'a.spec.ts',
+ 'b.spec.ts',
+ 'c.spec.ts',
+ 'scenarios/d.spec.ts',
+ 'f.spec.ts',
+ 'g.spec.ts',
+ ];
+ const perTest =
+ known.reduce((sum, file) => sum + recorded[file], 0) /
+ known.reduce((sum, file) => sum + suite[file], 0);
+ assert.equal(estimates.get('scenarios/e.spec.ts'), perTest);
+ assert.equal(estimates.has('removed.spec.ts'), false);
+ assert.equal(estimate(counts, {}).get('f.spec.ts'), 5);
+});
+
+test('the plan is deterministic and ties resolve by name and shard index', () => {
+ const counts = new Map([
+ ['x.spec.ts', 1],
+ ['y.spec.ts', 1],
+ ['z.spec.ts', 1],
+ ]);
+ const timings = { 'x.spec.ts': 5, 'y.spec.ts': 5, 'z.spec.ts': 5 };
+ const first = partition(estimate(counts, timings), 2);
+ const second = partition(estimate(new Map([...counts].reverse()), timings), 2);
+ assert.deepEqual(first, second);
+ assert.deepEqual(first.shards, [['x.spec.ts', 'z.spec.ts'], ['y.spec.ts']]);
+});
+
+test('a reservation hands the reserved shard less of the suite', () => {
+ const counts = listedFiles(listing(suite));
+ const estimates = estimate(counts, recorded);
+ const plain = partition(estimates, 3);
+ const reserved = partition(estimates, 3, [100, 0, 0]);
+ const seconds = (files) => files.reduce((sum, file) => sum + estimates.get(file), 0);
+ assert.ok(seconds(reserved.shards[0]) < seconds(plain.shards[0]));
+ assert.deepEqual(reserved.shards.flat().sort(), Object.keys(suite).sort());
+});
+
+test('graduated skips leave the pool before planning, and never empty it', (t) => {
+ const counts = listedFiles(listing(suite));
+ const skipped = applySkip(counts, 'specs/mock/b.spec.ts specs/mock/unknown.spec.ts');
+ assert.equal(skipped.dropped, 1);
+ assert.equal(skipped.counts.has('b.spec.ts'), false);
+ assert.equal(applySkip(counts, 'specs/mock/unknown.spec.ts').dropped, 0);
+ const everything = Object.keys(suite)
+ .map((file) => `specs/mock/${file}`)
+ .join(' ');
+ assert.equal(applySkip(counts, everything).counts, counts);
+
+ const seen = [];
+ for (const shard of ['1/2', '2/2']) {
+ const result = plan(t, {
+ files: suite,
+ timings: recorded,
+ shard,
+ skip: 'specs/mock/f.spec.ts',
+ });
+ assert.match(result.stdout, /codegraph-e2e-skip: dropped 1 graduated specs/);
+ seen.push(...result.files);
+ }
+ assert.deepEqual(
+ seen.sort(),
+ Object.keys(suite)
+ .filter((file) => file !== 'f.spec.ts')
+ .sort(),
+ );
+});
+
+test('a listing with load errors, or a malformed shard, fails the plan', (t) => {
+ assert.throws(
+ () => listedFiles({ errors: [{ message: 'SyntaxError' }], suites: [] }),
+ /SyntaxError/,
+ );
+ const dir = scratch(t);
+ writeFileSync(join(dir, 'list.json'), JSON.stringify(listing(suite)));
+ writeFileSync(join(dir, 'timings.json'), JSON.stringify({ files: recorded }));
+ for (const [shard, reserve] of [
+ ['0/2', ''],
+ ['3/2', ''],
+ ['x', ''],
+ ['1/2', '3:10'],
+ ]) {
+ const result = spawnSync(
+ process.execPath,
+ [
+ script,
+ 'plan',
+ '--list',
+ join(dir, 'list.json'),
+ '--timings',
+ join(dir, 'timings.json'),
+ '--shard',
+ shard,
+ '--reserve',
+ reserve,
+ '--out',
+ join(dir, 'out.txt'),
+ ],
+ { encoding: 'utf8' },
+ );
+ assert.notEqual(result.status, 0, `${shard} ${reserve}`);
+ }
+});
+
+test('record sums every attempt per file from JSON reports', (t) => {
+ const dir = scratch(t);
+ const report = (file, durations) => ({
+ suites: [
+ {
+ file,
+ specs: [],
+ suites: [
+ {
+ file,
+ specs: [{ tests: [{ results: durations.map((duration) => ({ duration })) }] }],
+ suites: [],
+ },
+ ],
+ },
+ ],
+ });
+ assert.deepEqual([...fileSeconds(report('a.spec.ts', [1000, 2500]))], [['a.spec.ts', 3.5]]);
+ mkdirSync(join(dir, 'memory-1', 'nested'), { recursive: true });
+ mkdirSync(join(dir, 'memory-2'));
+ writeFileSync(
+ join(dir, 'memory-1', 'nested', 'results.json'),
+ JSON.stringify(report('a.spec.ts', [1000])),
+ );
+ writeFileSync(
+ join(dir, 'memory-2', 'results.json'),
+ JSON.stringify(report('b.spec.ts', [2000, 500])),
+ );
+ writeFileSync(join(dir, 'memory-2', 'other.json'), JSON.stringify(report('c.spec.ts', [9000])));
+ const out = join(dir, 'timings.json');
+ const result = spawnSync(process.execPath, [script, 'record', '--out', out, dir], {
+ encoding: 'utf8',
+ });
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.deepEqual(JSON.parse(readFileSync(out, 'utf8')), {
+ files: { 'a.spec.ts': 1, 'b.spec.ts': 2.5 },
+ });
+});
+
+test('the committed timings are a flat map of spec files to seconds', () => {
+ const { files } = JSON.parse(
+ readFileSync(new URL('../../e2e/timings.mock.json', import.meta.url), 'utf8'),
+ );
+ const entries = Object.entries(files);
+ assert.ok(entries.length > 0);
+ for (const [file, seconds] of entries) {
+ assert.match(file, /\.spec\.ts$/);
+ assert.ok(Number.isFinite(seconds) && seconds >= 0, file);
+ }
+});
diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml
index bf8d7e431f..10085c52f0 100644
--- a/.github/workflows/playwright-mock.yml
+++ b/.github/workflows/playwright-mock.yml
@@ -70,7 +70,7 @@ jobs:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
CHANGED: ${{ github.event.pull_request.changed_files }}
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
- FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6","email_change":"true"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}'
+ FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}'
run: |
set +e
note() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
@@ -119,7 +119,7 @@ jobs:
INCLUDE="$FULL_INCLUDE"
note "| redis transport | run |"
fi
- if ! echo "$INCLUDE" | jq -e '.include | length >= 6' >/dev/null 2>&1; then
+ if ! echo "$INCLUDE" | jq -e '.include | length >= 10' >/dev/null 2>&1; then
note "_matrix assembly failed; running FULL_"
exit 0
fi
@@ -178,8 +178,8 @@ jobs:
(github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' &&
fromJSON(needs.codegraph_select.outputs.e2e_include)) ||
((github.event_name == 'pull_request' || inputs.coverage == 'pr') &&
- fromJSON('{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6","email_change":"true"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) ||
- fromJSON('{"include":[{"name":"memory, shard 1/2","stream_store":"memory","redis_image":"","suite":"full","shard":"1/2","artifact":"memory-1-of-2","email_change":"true"},{"name":"memory, shard 2/2","stream_store":"memory","redis_image":"","suite":"full","shard":"2/2","artifact":"memory-2-of-2"},{"name":"redis, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/2","artifact":"redis-1-of-2"},{"name":"redis, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/2","artifact":"redis-2-of-2"}]}')
+ fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) ||
+ fromJSON('{"include":[{"name":"memory, shard 1/10","stream_store":"memory","redis_image":"","suite":"full","shard":"1/10","artifact":"memory-1-of-10","email_change":"true"},{"name":"memory, shard 2/10","stream_store":"memory","redis_image":"","suite":"full","shard":"2/10","artifact":"memory-2-of-10"},{"name":"memory, shard 3/10","stream_store":"memory","redis_image":"","suite":"full","shard":"3/10","artifact":"memory-3-of-10"},{"name":"memory, shard 4/10","stream_store":"memory","redis_image":"","suite":"full","shard":"4/10","artifact":"memory-4-of-10"},{"name":"memory, shard 5/10","stream_store":"memory","redis_image":"","suite":"full","shard":"5/10","artifact":"memory-5-of-10"},{"name":"memory, shard 6/10","stream_store":"memory","redis_image":"","suite":"full","shard":"6/10","artifact":"memory-6-of-10"},{"name":"memory, shard 7/10","stream_store":"memory","redis_image":"","suite":"full","shard":"7/10","artifact":"memory-7-of-10"},{"name":"memory, shard 8/10","stream_store":"memory","redis_image":"","suite":"full","shard":"8/10","artifact":"memory-8-of-10"},{"name":"memory, shard 9/10","stream_store":"memory","redis_image":"","suite":"full","shard":"9/10","artifact":"memory-9-of-10"},{"name":"memory, shard 10/10","stream_store":"memory","redis_image":"","suite":"full","shard":"10/10","artifact":"memory-10-of-10"},{"name":"redis, shard 1/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/10","artifact":"redis-1-of-10"},{"name":"redis, shard 2/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/10","artifact":"redis-2-of-10"},{"name":"redis, shard 3/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"3/10","artifact":"redis-3-of-10"},{"name":"redis, shard 4/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"4/10","artifact":"redis-4-of-10"},{"name":"redis, shard 5/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"5/10","artifact":"redis-5-of-10"},{"name":"redis, shard 6/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"6/10","artifact":"redis-6-of-10"},{"name":"redis, shard 7/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"7/10","artifact":"redis-7-of-10"},{"name":"redis, shard 8/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"8/10","artifact":"redis-8-of-10"},{"name":"redis, shard 9/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"9/10","artifact":"redis-9-of-10"},{"name":"redis, shard 10/10","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"10/10","artifact":"redis-10-of-10"}]}')
}}
services:
redis:
@@ -223,7 +223,7 @@ jobs:
run: npm ci
- name: Test Playwright matrix selection
- run: node --test .github/scripts/playwright-matrix.test.mjs
+ run: node --test .github/scripts/playwright-matrix.test.mjs .github/scripts/playwright-shards.test.mjs
- name: Restore data-provider build cache
id: cache-data-provider
@@ -347,30 +347,31 @@ jobs:
env:
CI: 'true'
E2E_SKIP: ${{ needs.codegraph_select.outputs.e2e_skip }}
+ # Seconds of work memory shard 1 does after its suite (both registered-email profiles
+ # and the title ownership regressions), so the plan hands it that much less of the
+ # suite. Every lane of one store plans with the same value, so they agree.
+ E2E_SHARD_RESERVE: ${{ matrix.stream_store == 'memory' && '1:130' || '' }}
+ # Shards are planned by recorded duration, not test count: Playwright lists the suite,
+ # and .github/scripts/playwright-shards.mjs assigns whole spec files longest-first to
+ # the least-loaded shard, using e2e/timings.mock.json. Every shard derives the same
+ # disjoint partition of the listed files, so together they run exactly the suite.
+ #
+ # Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream) subtracts
+ # the earned skips from that listing before planning, so an unknown or stale name simply
+ # matches nothing; if subtraction would drop everything, nothing is skipped. Skipped specs
+ # still execute post-merge in every full-suite vote run, which is the net that catches a
+ # wrong skip.
run: |
- set +e
- # Graduated-spec skipping (dark until repo var CODEGRAPH_E2E_SKIP=on upstream): subtract
- # the earned skips from a run list derived from the tree itself, so an unknown or stale
- # name in the skip list simply matches nothing. If subtraction would drop everything —
- # or drops nothing — run the full shard exactly as before. Skipped specs still execute
- # post-merge in every full-suite vote run, which is the net that catches a wrong skip.
- RUN_ARGS=""
- if [ -n "$E2E_SKIP" ]; then
- KEEP=""; DROP=0
- for spec in $(git ls-files 'e2e/specs/mock/*.spec.ts' 'e2e/specs/mock/**/*.spec.ts' | sed 's|^e2e/||' | sort -u); do
- case "$spec" in *" "*) KEEP="$KEEP $spec"; continue;; esac
- case " $E2E_SKIP " in
- *" $spec "*) DROP=$((DROP+1));;
- *) KEEP="$KEEP $spec";;
- esac
- done
- if [ "$DROP" -gt 0 ] && [ -n "$KEEP" ]; then
- RUN_ARGS="$KEEP"
- echo "codegraph-e2e-skip: dropped $DROP graduated specs from this shard's pool"
- fi
- fi
- set -e
- npx playwright test --config=e2e/playwright.config.mock.ts --shard=${{ matrix.shard }} $RUN_ARGS
+ PLAYWRIGHT_JSON_OUTPUT_FILE=e2e/.generated/playwright-list.json \
+ npx playwright test --config=e2e/playwright.config.mock.ts --list --reporter=json > /dev/null
+ node .github/scripts/playwright-shards.mjs plan \
+ --list e2e/.generated/playwright-list.json \
+ --timings e2e/timings.mock.json \
+ --shard '${{ matrix.shard }}' \
+ --reserve "$E2E_SHARD_RESERVE" \
+ --skip "$E2E_SKIP" \
+ --out e2e/.generated/shard-tests.txt
+ npx playwright test --config=e2e/playwright.config.mock.ts --test-list=e2e/.generated/shard-tests.txt
- name: Run Redis stream transport e2e
if: matrix.suite == 'transport'
diff --git a/e2e/README.md b/e2e/README.md
index dc0592d5a3..64131b11a3 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -72,19 +72,31 @@ npm run e2e:mock:redis
Memory mode explicitly disables Redis. Redis mode defaults to database 15 with a `LibreChatE2E` key prefix, and fails closed: the test server pings Redis and verifies that the generation job manager did not silently fall back to memory. Override `REDIS_URI` or `E2E_REDIS_KEY_PREFIX` when needed.
-Pull request CI runs the complete mock suite in memory mode across three shards, plus a
-focused Redis transport suite. The Redis suite covers streaming fidelity, steering,
+Pull request CI runs the complete mock suite in memory mode across ten shards, plus a
+focused Redis transport suite across two. The Redis suite covers streaming fidelity, steering,
interrupts, resumptions, HITL approvals, completion, thread folding, model icons, and usage:
```sh
-npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/3
+npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/10
npm run e2e:mock:redis:transport
```
The nightly schedule and manual workflow dispatch run the complete mock suite in both stream
-modes across two shards per mode. Every shard keeps one worker so tests do not contend for its
+modes across ten shards per mode. Every shard keeps one worker so tests do not contend for its
authenticated user and database.
+CI splits the complete suite by recorded duration rather than test count: each shard lists the
+suite, and `.github/scripts/playwright-shards.mjs plan` hands out whole spec files longest-first
+to the least-loaded shard using `e2e/timings.mock.json`. A spec with no recorded time is
+estimated from its test count, so new specs need nothing; refresh the timings when the shards
+drift apart. Each shard's report artifact carries a `results.json`, so download a recent run's
+`playwright-report-memory-*` artifacts and rebuild the file from them:
+
+```sh
+gh run download <run-id> --pattern 'playwright-report-memory-*' --dir /tmp/e2e-reports
+node .github/scripts/playwright-shards.mjs record --out e2e/timings.mock.json /tmp/e2e-reports
+```
+
## Property-based browser testing
Bombadil explores randomized sequences across the core chat loop, message branches,
diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts
index a579fcabcc..2240718ef1 100644
--- a/e2e/playwright.config.mock.ts
+++ b/e2e/playwright.config.mock.ts
@@ -516,8 +516,15 @@ export default defineConfig({
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 2 : 0,
workers: 1,
+ /** In CI the JSON results land beside the HTML report (reporters end in order, after the HTML
+ * reporter has replaced its folder), so the report artifact also carries the per-file
+ * timings `.github/scripts/playwright-shards.mjs record` rebuilds the shard plan from. */
reporter: process.env.CI
- ? [['html', { outputFolder: reportPath, open: 'never' }], ['line']]
+ ? [
+ ['html', { outputFolder: reportPath, open: 'never' }],
+ ['line'],
+ ['json', { outputFile: path.join(reportPath, 'results.json') }],
+ ]
: [['html', { outputFolder: reportPath }], ['list']],
use: {
baseURL,
diff --git a/e2e/timings.mock.json b/e2e/timings.mock.json
new file mode 100644
index 0000000000..c8b1242749
--- /dev/null
+++ b/e2e/timings.mock.json
@@ -0,0 +1,325 @@
+{
+ "files": {
+ "activity-fold.spec.ts": 20.2,
+ "activity-labels.spec.ts": 28.3,
+ "activity-phases.spec.ts": 6,
+ "activity-prose.spec.ts": 25.3,
+ "activity-thought.spec.ts": 10.2,
+ "agent-collaboration-tools.spec.ts": 8.3,
+ "agent-file-ownership.spec.ts": 15,
+ "agent-handoffs.spec.ts": 77.2,
+ "agent-management-files.spec.ts": 0.4,
+ "agent-skills-added.spec.ts": 5.5,
+ "agent-skills.spec.ts": 53.5,
+ "agents.spec.ts": 8.5,
+ "app-load.spec.ts": 2,
+ "attachment-preview-loop.spec.ts": 5.3,
+ "auth.spec.ts": 10.2,
+ "background-tools.spec.ts": 8.7,
+ "bookmarks.spec.ts": 15.6,
+ "chat.spec.ts": 41.6,
+ "code-upload.spec.ts": 0,
+ "completion.spec.ts": 4,
+ "composer-upload-layout.spec.ts": 10.9,
+ "content-filters.persisted.spec.ts": 19.2,
+ "content-filters.submissions.spec.ts": 1.2,
+ "conversation-management.spec.ts": 101,
+ "deferred-tools-hitl.spec.ts": 9.8,
+ "deployment-skills.spec.ts": 2.3,
+ "enforced-model-specs.spec.ts": 0,
+ "export.spec.ts": 16.2,
+ "file-provisioning.spec.ts": 22.7,
+ "hover-actions.spec.ts": 26.5,
+ "idle-animations.spec.ts": 4.9,
+ "isolation.spec.ts": 13,
+ "mcp-allowlist-override.spec.ts": 0.2,
+ "mcp-apps.spec.ts": 0,
+ "mcp-ephemeral.spec.ts": 11.7,
+ "mcp-fixture-isolation.spec.ts": 0,
+ "mcp-oauth-readiness.spec.ts": 57.9,
+ "mcp-oauth-resume.spec.ts": 7.8,
+ "mcp-tool-list-changed.spec.ts": 0,
+ "mcp.spec.ts": 7.6,
+ "memory.spec.ts": 3.9,
+ "mermaid-artifacts.spec.ts": 31.1,
+ "message-edit-layout.spec.ts": 8.7,
+ "message-tree.spec.ts": 116.3,
+ "message-visual.spec.ts": 118,
+ "model-replay-tools.spec.ts": 4.8,
+ "model-replay.spec.ts": 4.2,
+ "model-spec-branding.spec.ts": 7.8,
+ "model-spec-icons.spec.ts": 12.6,
+ "model-spec-skills.spec.ts": 4,
+ "model-spec-starters.spec.ts": 9.4,
+ "model-switching.spec.ts": 8.6,
+ "passkeys.spec.ts": 6.7,
+ "permissions.spec.ts": 0.3,
+ "private-text.spec.ts": 43.3,
+ "projects.spec.ts": 44.8,
+ "prompts.spec.ts": 6,
+ "queued-reveal.spec.ts": 27.8,
+ "quotes.spec.ts": 139.7,
+ "run-files-delivery.spec.ts": 28.8,
+ "run-files-lifecycle.spec.ts": 57.3,
+ "run-files.spec.ts": 21.9,
+ "scenarios/account-deletion.spec.ts": 0.3,
+ "scenarios/action-method-labels.spec.ts": 20.7,
+ "scenarios/adopted-intents.spec.ts": 13.4,
+ "scenarios/agent-builder-header.spec.ts": 13.2,
+ "scenarios/agent-conversation-starters.spec.ts": 46.4,
+ "scenarios/agent-detail-focus-handoff.spec.ts": 4,
+ "scenarios/agent-detail-morph.spec.ts": 21.6,
+ "scenarios/agent-grid-columns.spec.ts": 3.6,
+ "scenarios/agent-search-clear.spec.ts": 2.6,
+ "scenarios/agent-selector-caps.spec.ts": 23.7,
+ "scenarios/agent-version-projection.spec.ts": 14.9,
+ "scenarios/approval-provenance.spec.ts": 5.8,
+ "scenarios/artifact-edit-refetch.spec.ts": 9.5,
+ "scenarios/artifact-trigger-rows.spec.ts": 7.7,
+ "scenarios/ask-question-refine.spec.ts": 31.7,
+ "scenarios/assistant-select-empty.spec.ts": 8.6,
+ "scenarios/auth-control-shape.spec.ts": 15.1,
+ "scenarios/auth-log-redaction.spec.ts": 0.1,
+ "scenarios/auto-expand-tools-setting.spec.ts": 7.9,
+ "scenarios/avatar-glyph-contrast.spec.ts": 5.9,
+ "scenarios/badge-label.spec.ts": 7.8,
+ "scenarios/button-roles.spec.ts": 11.2,
+ "scenarios/category-filter-state.spec.ts": 5.1,
+ "scenarios/chat-context.spec.ts": 22.8,
+ "scenarios/chat-facade-live-attachments.spec.ts": 42.7,
+ "scenarios/chat-facade-transport.spec.ts": 5.3,
+ "scenarios/chat-filter-menu.spec.ts": 92.4,
+ "scenarios/chat-host-config.spec.ts": 4.5,
+ "scenarios/chat-package.spec.ts": 20.3,
+ "scenarios/chat-settings-owners.spec.ts": 12.9,
+ "scenarios/chrome-border.spec.ts": 13.2,
+ "scenarios/chrome-quiet-surfaces.spec.ts": 13.6,
+ "scenarios/clickhouse-checkbox.spec.ts": 21.6,
+ "scenarios/clickhouse-control-radius.spec.ts": 10.5,
+ "scenarios/clickhouse-layers.spec.ts": 11.9,
+ "scenarios/clickhouse-spacing-mono.spec.ts": 9.9,
+ "scenarios/clickhouse-switch.spec.ts": 11.8,
+ "scenarios/clickhouse-table.spec.ts": 27.5,
+ "scenarios/clickhouse-theme-shape.spec.ts": 20.7,
+ "scenarios/clickhouse-theme.spec.ts": 26.1,
+ "scenarios/code-window-header.spec.ts": 5.5,
+ "scenarios/code-workspace-composer.spec.ts": 26.5,
+ "scenarios/cold-load-clearance.spec.ts": 8.1,
+ "scenarios/compaction-abort-finalize.spec.ts": 21.8,
+ "scenarios/compaction-collision-settlement.spec.ts": 26.5,
+ "scenarios/compaction-rerun-controls.spec.ts": 55.9,
+ "scenarios/composer-context.spec.ts": 17.2,
+ "scenarios/composer-defaults.spec.ts": 25.3,
+ "scenarios/composer-numeric-reasoning.spec.ts": 6.6,
+ "scenarios/composer-palette.spec.ts": 29.3,
+ "scenarios/composer-pane-controls.spec.ts": 14.4,
+ "scenarios/composer-position.spec.ts": 13.9,
+ "scenarios/composer-queue.spec.ts": 69.9,
+ "scenarios/composer-redesign.spec.ts": 44.3,
+ "scenarios/composer-surfaces.spec.ts": 4.6,
+ "scenarios/composer-tool-approval.spec.ts": 23.8,
+ "scenarios/config-override-validation.spec.ts": 0.5,
+ "scenarios/configured-footer-clearance.spec.ts": 2.8,
+ "scenarios/configured-footer.spec.ts": 4.1,
+ "scenarios/content-handler-message-refs.spec.ts": 7.1,
+ "scenarios/contract.spec.ts": 10.7,
+ "scenarios/contrast-switch-position.spec.ts": 3.9,
+ "scenarios/control-border.spec.ts": 53.5,
+ "scenarios/control-spacing.spec.ts": 15.7,
+ "scenarios/conversation-list-filters.spec.ts": 14.5,
+ "scenarios/corner-roles.spec.ts": 8.8,
+ "scenarios/css-color-triplets.spec.ts": 0.2,
+ "scenarios/data-table-row-estimate.spec.ts": 4.4,
+ "scenarios/denied-web-storage.spec.ts": 6,
+ "scenarios/deployment-theme-first-paint.spec.ts": 30.3,
+ "scenarios/deployment-theme.spec.ts": 9.1,
+ "scenarios/design-rule-enforcement.spec.ts": 14.5,
+ "scenarios/dialog-chrome.spec.ts": 10.1,
+ "scenarios/dialog-scrim-role.spec.ts": 17.3,
+ "scenarios/disabled-recipe.spec.ts": 9.6,
+ "scenarios/displaced-artifact-buffer.spec.ts": 27.1,
+ "scenarios/drawer-edge-theme.spec.ts": 5,
+ "scenarios/dropped-classes.spec.ts": 4.5,
+ "scenarios/email-change-gating.spec.ts": 12.1,
+ "scenarios/empty-prompts-panel.spec.ts": 2.7,
+ "scenarios/ephemeral-retention.spec.ts": 19.5,
+ "scenarios/failed-summary-history.spec.ts": 26.5,
+ "scenarios/field-fill.spec.ts": 17.1,
+ "scenarios/field-focus.spec.ts": 12.9,
+ "scenarios/field-variants.spec.ts": 51.4,
+ "scenarios/file-tile-theme.spec.ts": 12.4,
+ "scenarios/files-tree-removal.spec.ts": 3.4,
+ "scenarios/focus-outline.spec.ts": 44.5,
+ "scenarios/focus-primitives.spec.ts": 20.1,
+ "scenarios/focus-role-utilities.spec.ts": 25,
+ "scenarios/focus-subtle.spec.ts": 14.8,
+ "scenarios/footer-shell-answer.spec.ts": 5.2,
+ "scenarios/grouped-model-params.spec.ts": 20.4,
+ "scenarios/header-action-buttons.spec.ts": 5.6,
+ "scenarios/header-control-surface.spec.ts": 11.1,
+ "scenarios/highlight-throttle.spec.ts": 36.8,
+ "scenarios/input-number-disabled.spec.ts": 7.3,
+ "scenarios/input-number-option-focus.spec.ts": 19.5,
+ "scenarios/input-number-reset.spec.ts": 4.6,
+ "scenarios/lazy-mermaid.spec.ts": 8.9,
+ "scenarios/lia-art.spec.ts": 1.5,
+ "scenarios/lia-engine.spec.ts": 2,
+ "scenarios/lia-mascot.spec.ts": 83.1,
+ "scenarios/literal-roles-theme.spec.ts": 14.5,
+ "scenarios/local-file-saves.spec.ts": 0.4,
+ "scenarios/locked-registry.spec.ts": 0,
+ "scenarios/long-reasoning-stream.spec.ts": 13.1,
+ "scenarios/marketplace-mine.spec.ts": 8,
+ "scenarios/marketplace-recovery.spec.ts": 53.8,
+ "scenarios/marketplace-sorting.spec.ts": 9.8,
+ "scenarios/marketplace-start-chat.spec.ts": 4.1,
+ "scenarios/mcp-app-operation-limits.spec.ts": 0.1,
+ "scenarios/mcp-oauth-tiles.spec.ts": 3,
+ "scenarios/mcp-option-series-color.spec.ts": 21.4,
+ "scenarios/media-overlay-theme.spec.ts": 17.3,
+ "scenarios/menu-shadow.spec.ts": 13.8,
+ "scenarios/mermaid-export-menu.spec.ts": 6.7,
+ "scenarios/mermaid-source-download.spec.ts": 14.2,
+ "scenarios/message-actions.spec.ts": 4.3,
+ "scenarios/message-parts-host.spec.ts": 21.9,
+ "scenarios/message-text-container.spec.ts": 4.3,
+ "scenarios/message-theme.spec.ts": 25,
+ "scenarios/mobile-composer-edges.spec.ts": 2.2,
+ "scenarios/mobile-drawer-banner.spec.ts": 7.7,
+ "scenarios/mobile-drawer-controls.spec.ts": 36.1,
+ "scenarios/model-selector-search.spec.ts": 12.1,
+ "scenarios/mouse-reveal.spec.ts": 8.6,
+ "scenarios/my-files-virtual.spec.ts": 9.1,
+ "scenarios/native-search-warning.spec.ts": 0,
+ "scenarios/nested-popovers.spec.ts": 32.5,
+ "scenarios/new-chat-panel-preference.spec.ts": 11.8,
+ "scenarios/page-surfaces.spec.ts": 12.1,
+ "scenarios/palette-show-all.spec.ts": 36.1,
+ "scenarios/parts.spec.ts": 13.4,
+ "scenarios/passkeys.spec.ts": 59.1,
+ "scenarios/pending-artifact-row-slot.spec.ts": 5.1,
+ "scenarios/persisted-context-usage.spec.ts": 11.6,
+ "scenarios/pinned-drag-rules.spec.ts": 22.6,
+ "scenarios/pinned-row-controls.spec.ts": 22.7,
+ "scenarios/pinned-row-menus.spec.ts": 13.2,
+ "scenarios/plain-text-artifact-row.spec.ts": 3.5,
+ "scenarios/pointerless-reveal.spec.ts": 3.8,
+ "scenarios/policy-links-placement.spec.ts": 11.1,
+ "scenarios/primitive-sizes.spec.ts": 11.7,
+ "scenarios/principal-type-labels.spec.ts": 24.1,
+ "scenarios/project-access.spec.ts": 9.9,
+ "scenarios/project-context.spec.ts": 9.7,
+ "scenarios/project-files-ui.spec.ts": 12.7,
+ "scenarios/project-policy.spec.ts": 1.4,
+ "scenarios/project-resume.spec.ts": 1.5,
+ "scenarios/project-rows-spaced.spec.ts": 3.5,
+ "scenarios/projects-dialogs.spec.ts": 30.1,
+ "scenarios/prompt-category-icon.spec.ts": 21.2,
+ "scenarios/prompt-row-permissions.spec.ts": 15.4,
+ "scenarios/prompt-surfaces.spec.ts": 12.2,
+ "scenarios/prose-shape.spec.ts": 11.3,
+ "scenarios/published-consumer-appearance.spec.ts": 1.9,
+ "scenarios/published-consumer-content.spec.ts": 0,
+ "scenarios/published-font-faces.spec.ts": 4.7,
+ "scenarios/published-package-peers.spec.ts": 6.6,
+ "scenarios/published-preset-animations.spec.ts": 0.2,
+ "scenarios/published-theme-parity.spec.ts": 3.1,
+ "scenarios/published-theme-stylesheet.spec.ts": 0,
+ "scenarios/pull-request-enterprise-host.spec.ts": 5.2,
+ "scenarios/queue-owners.spec.ts": 7.3,
+ "scenarios/queue-row-actions.spec.ts": 60.5,
+ "scenarios/quieter-panels.spec.ts": 34.5,
+ "scenarios/rail-nav-states.spec.ts": 21.5,
+ "scenarios/redis-stream.spec.ts": 5.5,
+ "scenarios/registration-consent.spec.ts": 10.2,
+ "scenarios/reload-resume.spec.ts": 34.5,
+ "scenarios/reply-read-state.spec.ts": 12,
+ "scenarios/reset-password.spec.ts": 0,
+ "scenarios/resume-body-end.spec.ts": 15.7,
+ "scenarios/run-indicators.spec.ts": 25.2,
+ "scenarios/saml-glyph.spec.ts": 1.3,
+ "scenarios/scrim-opacity.spec.ts": 11.9,
+ "scenarios/semantic-surfaces.spec.ts": 41.1,
+ "scenarios/series-eight-theme.spec.ts": 7.3,
+ "scenarios/share-deployment-theme.spec.ts": 44.7,
+ "scenarios/shared-undocked-artifacts.spec.ts": 9.1,
+ "scenarios/short-artifact-type.spec.ts": 5.7,
+ "scenarios/sidebar-active-row.spec.ts": 3.6,
+ "scenarios/sidebar-row-action-fills.spec.ts": 8.1,
+ "scenarios/sidebar-single-scroll.spec.ts": 31.2,
+ "scenarios/signin-controls.spec.ts": 9.7,
+ "scenarios/skills-picker-narrow-header.spec.ts": 14.3,
+ "scenarios/stale-gutter.spec.ts": 2.9,
+ "scenarios/state-roles.spec.ts": 14.9,
+ "scenarios/step-reducer.spec.ts": 11.2,
+ "scenarios/streaming-markdown-blocks.spec.ts": 15,
+ "scenarios/stylesheet-roles-theme.spec.ts": 7.2,
+ "scenarios/subagent-context-total.spec.ts": 6.8,
+ "scenarios/suppression-baseline.spec.ts": 246.6,
+ "scenarios/temporary-chat-icon.spec.ts": 2.7,
+ "scenarios/theme-override.spec.ts": 0.4,
+ "scenarios/theme-paint.spec.ts": 17.6,
+ "scenarios/theme-radius-font-scale.spec.ts": 15.3,
+ "scenarios/theme-root-nesting.spec.ts": 9.3,
+ "scenarios/theme-shadow-scale.spec.ts": 15.8,
+ "scenarios/theme-surfaces.spec.ts": 27.2,
+ "scenarios/theme-sweep-roles.spec.ts": 13.3,
+ "scenarios/theme-token-utilities.spec.ts": 10.8,
+ "scenarios/theme-tooltip.spec.ts": 10.7,
+ "scenarios/theme-unknown-appearance.spec.ts": 17,
+ "scenarios/thought-peek-tools.spec.ts": 31.8,
+ "scenarios/thoughts-floating-bar.spec.ts": 27.5,
+ "scenarios/tool-allow-always.spec.ts": 7.2,
+ "scenarios/tool-approval-invalid-json.spec.ts": 6.3,
+ "scenarios/tool-context-split.spec.ts": 74.8,
+ "scenarios/tool-library-small-screen.spec.ts": 33.2,
+ "scenarios/tool-output-pane.spec.ts": 5.7,
+ "scenarios/tool-output-polish.spec.ts": 9.2,
+ "scenarios/tool-steps.spec.ts": 16.5,
+ "scenarios/touch-field-zoom.spec.ts": 5.7,
+ "scenarios/turbo-cache.spec.ts": 1.1,
+ "scenarios/two-factor-deep-link.spec.ts": 13.5,
+ "scenarios/two-factor-required.spec.ts": 13.6,
+ "scenarios/two-in-one-reveal.spec.ts": 6.1,
+ "scenarios/type-roles.spec.ts": 8.4,
+ "scenarios/type-steps.spec.ts": 6.3,
+ "scenarios/typecheck.spec.ts": 1.3,
+ "scenarios/ui-scale-layout.spec.ts": 20,
+ "scenarios/ui-scale-persistence.spec.ts": 6.8,
+ "scenarios/ui-scale-stepper.spec.ts": 15.1,
+ "scenarios/undocked-artifact-generation.spec.ts": 18.5,
+ "scenarios/undocked-artifact-save-rejection.spec.ts": 10.3,
+ "scenarios/undocked-artifacts-blocked-popup.spec.ts": 4.7,
+ "scenarios/undocked-artifacts.spec.ts": 67.7,
+ "scenarios/unknown-classes.spec.ts": 14.3,
+ "scenarios/unseen-replies.spec.ts": 106.1,
+ "scenarios/v4-focus.spec.ts": 11.5,
+ "scenarios/v4-rendering.spec.ts": 18.7,
+ "scenarios/verified-tool-mark.spec.ts": 16.5,
+ "scenarios/welcome-disclaimer.spec.ts": 6.4,
+ "scenarios/yaml-rate-limits.spec.ts": 7.8,
+ "scenarios/yaml-theme-fallback.spec.ts": 15,
+ "schedules-execution.spec.ts": 124.9,
+ "schedules.spec.ts": 4.3,
+ "shared-links.spec.ts": 15.2,
+ "shortcuts.spec.ts": 11.3,
+ "sidebar.spec.ts": 8.1,
+ "skill-file-authoring.spec.ts": 7.9,
+ "soft-default.spec.ts": 58.3,
+ "stateful-code-bridge.spec.ts": 0,
+ "steering-escalation.spec.ts": 57.6,
+ "steering.spec.ts": 188.5,
+ "streaming.spec.ts": 5.5,
+ "subagent-activity.spec.ts": 20.5,
+ "subagent-results.spec.ts": 5.9,
+ "thread-fold.spec.ts": 51.3,
+ "thread-renderers.spec.ts": 28.9,
+ "tool-approval-modes.spec.ts": 39.3,
+ "tool-approvals.spec.ts": 81.7,
+ "two-factor-required.spec.ts": 11.2,
+ "two-factor.spec.ts": 13.4,
+ "unified-upload.spec.ts": 16.6,
+ "usage.spec.ts": 36.4,
+ "user-image-alignment.spec.ts": 5.4
+ }
+}