The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor: balance serialized server shards by recorded suite duration (#11528)
Devin Foley49217aadf0af
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.ymlindex 925c520ba4..7fe5fca58e 100644--- a/.github/workflows/pr.yml+++ b/.github/workflows/pr.yml@@ -287,9 +287,11 @@ jobs:fail-fast: falsematrix:include:- # A successful PR run on 2026-08-04 (30876682788) spent 256s in- # serialized shard 2/4, making its 305s job the run's slowest check.- # Five shards reduce the measured 739s suite total to about 148s per+ # A successful PR run on 2026-08-17 (32012408876) spent 291s in+ # serialized shard 1/5 while its siblings ran 170-201s: round-robin+ # clustered the heavy suites on one runner. Shards are now balanced+ # by recorded duration (scripts/serialized-shard-durations.json),+ # which levels the measured 968s suite total to about 194s per# runner before setup overhead.- shard_index: 0shard_count: 5diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjsindex ea74a3de12..690de987a3 100644--- a/scripts/__tests__/run-vitest-stable-shard.test.mjs+++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs@@ -13,6 +13,11 @@ import {const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");const script = path.join(repoRoot, "scripts", "run-vitest-stable.mjs");const durationsManifest = path.join(repoRoot, "scripts", "general-server-shard-durations.json");+const serializedDurationsManifest = path.join(+ repoRoot,+ "scripts",+ "serialized-shard-durations.json",+);function dryRun(args) {const result = spawnSync(process.execPath, [script, ...args, "--dry-run"], {@@ -167,6 +172,39 @@ test("the checked-in manifest loads and covers most of the current suite set", ();});+test("the checked-in serialized manifest loads and covers most of the current suite set", () => {+ const durations = loadShardDurations(serializedDurationsManifest);+ assert.ok(Object.keys(durations).length > 0, "manifest must parse to a non-empty duration map");++ const shard = dryRunJson(["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"]);+ const currentFiles = shard.selectedSerializedSuites;+ const known = currentFiles.filter((file) => durations[file] !== undefined).length;+ assert.ok(+ known / currentFiles.length >= 0.5,+ `manifest is stale: only ${known} of ${currentFiles.length} suites have recorded durations — regenerate it from a recent PR run (see the manifest's $comment)`,+ );+});++test("the real serialized shard partition is duration-balanced", () => {+ const durations = loadShardDurations(serializedDurationsManifest);+ const fallback = defaultSuiteWeight(durations);+ const shards = Array.from({ length: SERIALIZED_SHARD_COUNT }, (_, index) =>+ dryRunJson(["--mode", "serialized", "--shard-index", String(index), "--shard-count", String(SERIALIZED_SHARD_COUNT)]),+ );++ const totals = shards.map((shard) =>+ shard.selectedSerializedSuites.reduce((sum, file) => sum + (durations[file] ?? fallback), 0),+ );+ const maxTotal = Math.max(...totals);+ const minTotal = Math.min(...totals);+ // LPT keeps the spread within the heaviest single suite; use that as the bound.+ const heaviest = Math.max(...Object.values(durations));+ assert.ok(+ maxTotal - minTotal <= heaviest,+ `serialized shard weight spread ${maxTotal - minTotal}ms exceeds heaviest suite ${heaviest}ms: ${totals.join(", ")}`,+ );+});+test("the real shard partition is duration-balanced", () => {const durations = loadShardDurations(durationsManifest);const fallback = defaultSuiteWeight(durations);diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjsindex b77c9d8a38..4f7af35bd4 100644--- a/scripts/run-vitest-stable.mjs+++ b/scripts/run-vitest-stable.mjs@@ -11,6 +11,9 @@ const scriptsDir = path.dirname(fileURLToPath(import.meta.url));const generalServerShardDurations = loadShardDurations(path.join(scriptsDir, "general-server-shard-durations.json"),);+const serializedShardDurations = loadShardDurations(+ path.join(scriptsDir, "serialized-shard-durations.json"),+);const serverRoot = path.join(repoRoot, "server");const serverSrcDir = path.join(repoRoot, "server", "src");const serverTestsDir = path.join(repoRoot, "server", "src", "__tests__");@@ -251,7 +254,18 @@ function parseCliOptions(argv) {}function selectSerializedSuites(routeTests, shardIndex, shardCount) {- return routeTests.filter((_, index) => index % shardCount === shardIndex);+ // Same duration-aware LPT partition as the general-server lane. Round-robin+ // over the alphabetical list clustered the heavy heartbeat/issues suites on+ // one shard (291s vs 170-201s test steps across the matrix in actions run+ // 32012408876), which made that shard the whole PR run's slowest check.+ const byRepoPath = new Map(routeTests.map((routeTest) => [routeTest.repoPath, routeTest]));+ const shardFiles = selectGeneralServerShard(+ routeTests.map((routeTest) => routeTest.repoPath),+ shardIndex,+ shardCount,+ serializedShardDurations,+ );+ return shardFiles.map((file) => byRepoPath.get(file));}function runVitest(args, label) {diff --git a/scripts/serialized-shard-durations.json b/scripts/serialized-shard-durations.jsonnew file mode 100644index 0000000000..4e6fe71ee7--- /dev/null+++ b/scripts/serialized-shard-durations.json@@ -0,0 +1,140 @@+{+ "$comment": "Per-suite wall-clock durations (ms) for the serialized route/authz vitest lane, used by scripts/run-vitest-stable.mjs to balance suites across the PR shard matrix. Sampled from a real PR run of .github/workflows/pr.yml (actions run 32012408876, 2026-08-17) by diffing consecutive '[test:run] <suite>' label timestamps in the 'Run serialized server test shard' logs - that captures each suite's true serial cost including the per-suite vitest spawn overhead, not just the vitest-reported test time. Suites missing here get the median weight, so the manifest only needs occasional refreshes.",+ "unit": "ms",+ "durations": {+ "server/src/__tests__/access-routes-permissions-upgrade.test.ts": 11105,+ "server/src/__tests__/activity-routes.test.ts": 3181,+ "server/src/__tests__/adapter-model-refresh-routes.test.ts": 3934,+ "server/src/__tests__/adapter-routes-authz.test.ts": 4391,+ "server/src/__tests__/adapter-routes.test.ts": 5343,+ "server/src/__tests__/agent-action-audit-routes.test.ts": 11458,+ "server/src/__tests__/agent-adapter-validation-routes.test.ts": 5758,+ "server/src/__tests__/agent-cross-tenant-authz-routes.test.ts": 3818,+ "server/src/__tests__/agent-device-login-routes.test.ts": 6486,+ "server/src/__tests__/agent-instructions-routes.test.ts": 5532,+ "server/src/__tests__/agent-live-run-routes.test.ts": 5498,+ "server/src/__tests__/agent-permissions-routes.test.ts": 8978,+ "server/src/__tests__/agent-secrets-routes.test.ts": 11780,+ "server/src/__tests__/agent-skills-routes.test.ts": 7404,+ "server/src/__tests__/agent-test-environment-routes.test.ts": 5845,+ "server/src/__tests__/approval-routes-idempotency.test.ts": 6142,+ "server/src/__tests__/assets.test.ts": 3555,+ "server/src/__tests__/auth-routes.test.ts": 2094,+ "server/src/__tests__/auth-session-route.test.ts": 3070,+ "server/src/__tests__/authz-company-access.test.ts": 2893,+ "server/src/__tests__/authz-existence-oracle-guard.test.ts": 1121,+ "server/src/__tests__/authz-secret-context.test.ts": 2398,+ "server/src/__tests__/board-chat-route-feature-flag.test.ts": 1032,+ "server/src/__tests__/bootstrap-claim-routes.test.ts": 4748,+ "server/src/__tests__/built-in-agent-routes.test.ts": 4015,+ "server/src/__tests__/cases-routes.test.ts": 12174,+ "server/src/__tests__/cli-auth-routes.test.ts": 5228,+ "server/src/__tests__/cloud-routes.test.ts": 2146,+ "server/src/__tests__/companies-route-cross-company-authz.test.ts": 4314,+ "server/src/__tests__/companies-route-path-guard.test.ts": 3004,+ "server/src/__tests__/company-branding-route.test.ts": 3920,+ "server/src/__tests__/company-import-transfer-routes.test.ts": 8905,+ "server/src/__tests__/company-portability-routes.test.ts": 3034,+ "server/src/__tests__/company-portability.test.ts": 4800,+ "server/src/__tests__/company-search-extract-routes.test.ts": 7621,+ "server/src/__tests__/company-search-rate-limit-routes.test.ts": 7587,+ "server/src/__tests__/company-skill-policy-routes.test.ts": 6894,+ "server/src/__tests__/company-skills-import-authz-routes.test.ts": 12068,+ "server/src/__tests__/company-skills-routes.test.ts": 7484,+ "server/src/__tests__/company-user-directory-route.test.ts": 4728,+ "server/src/__tests__/costs-service.test.ts": 8319,+ "server/src/__tests__/decision-queues-routes.test.ts": 6956,+ "server/src/__tests__/document-annotation-routes.test.ts": 4375,+ "server/src/__tests__/environment-custom-image-routes.test.ts": 4387,+ "server/src/__tests__/environment-instance-routes.test.ts": 4527,+ "server/src/__tests__/environment-routes.test.ts": 4780,+ "server/src/__tests__/environment-selection-route-guards.test.ts": 4208,+ "server/src/__tests__/execution-workspaces-routes.test.ts": 3450,+ "server/src/__tests__/express5-auth-wildcard.test.ts": 1105,+ "server/src/__tests__/external-object-routes.test.ts": 6231,+ "server/src/__tests__/folders-routes.test.ts": 2802,+ "server/src/__tests__/health-dev-server-token.test.ts": 2541,+ "server/src/__tests__/health.test.ts": 2090,+ "server/src/__tests__/heartbeat-dependency-scheduling.test.ts": 14289,+ "server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts": 13747,+ "server/src/__tests__/heartbeat-process-recovery.test.ts": 55336,+ "server/src/__tests__/inbox-agent-policy-routes.test.ts": 10057,+ "server/src/__tests__/inbox-archive-routes.test.ts": 9644,+ "server/src/__tests__/instance-database-backups-routes.test.ts": 2962,+ "server/src/__tests__/instance-settings-routes.test.ts": 5356,+ "server/src/__tests__/invite-accept-existing-member.test.ts": 4812,+ "server/src/__tests__/invite-accept-gateway-defaults.test.ts": 10423,+ "server/src/__tests__/invite-accept-replay.test.ts": 5116,+ "server/src/__tests__/invite-create-route.test.ts": 4640,+ "server/src/__tests__/invite-expiry.test.ts": 7291,+ "server/src/__tests__/invite-join-manager.test.ts": 7124,+ "server/src/__tests__/invite-list-route.test.ts": 7580,+ "server/src/__tests__/invite-logo-route.test.ts": 5104,+ "server/src/__tests__/invite-onboarding-text.test.ts": 7095,+ "server/src/__tests__/invite-rate-limit-route.test.ts": 7446,+ "server/src/__tests__/invite-summary-route.test.ts": 7598,+ "server/src/__tests__/invite-test-resolution-route.test.ts": 6135,+ "server/src/__tests__/invite-url-public-base-url.test.ts": 3482,+ "server/src/__tests__/issue-activity-events-routes.test.ts": 6254,+ "server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts": 13283,+ "server/src/__tests__/issue-assigned-backlog-contract-routes.test.ts": 5442,+ "server/src/__tests__/issue-assignee-invokability-routes.test.ts": 4602,+ "server/src/__tests__/issue-attachment-routes.test.ts": 5267,+ "server/src/__tests__/issue-blocker-diagnostics-routes.test.ts": 10902,+ "server/src/__tests__/issue-closed-workspace-routes.test.ts": 6589,+ "server/src/__tests__/issue-comment-attribution-audit-routes.test.ts": 11053,+ "server/src/__tests__/issue-comment-cancel-routes.test.ts": 4687,+ "server/src/__tests__/issue-comment-reopen-routes.test.ts": 4009,+ "server/src/__tests__/issue-create-deduplication-routes.test.ts": 11885,+ "server/src/__tests__/issue-dependency-wakeups-routes.test.ts": 5701,+ "server/src/__tests__/issue-document-restore-routes.test.ts": 5902,+ "server/src/__tests__/issue-execution-policy-routes.test.ts": 6571,+ "server/src/__tests__/issue-feedback-routes.test.ts": 3978,+ "server/src/__tests__/issue-identifier-routes.test.ts": 11276,+ "server/src/__tests__/issue-list-assignee-filter-routes.test.ts": 12518,+ "server/src/__tests__/issue-list-updatedsince-filter-routes.test.ts": 10919,+ "server/src/__tests__/issue-onboarding-first-task-routes.test.ts": 12493,+ "server/src/__tests__/issue-scheduled-retry-routes.test.ts": 11559,+ "server/src/__tests__/issue-stale-execution-lock-routes.test.ts": 11281,+ "server/src/__tests__/issue-stalled-review-decision-routes.test.ts": 11437,+ "server/src/__tests__/issue-subtree-diagnostics-routes.test.ts": 11043,+ "server/src/__tests__/issue-telemetry-routes.test.ts": 4745,+ "server/src/__tests__/issue-thread-interaction-routes.test.ts": 8825,+ "server/src/__tests__/issue-tree-control-routes.test.ts": 2689,+ "server/src/__tests__/issue-update-comment-wakeup-routes.test.ts": 6552,+ "server/src/__tests__/issue-wake-diagnostics-routes.test.ts": 11098,+ "server/src/__tests__/issue-watchdogs-routes.test.ts": 15909,+ "server/src/__tests__/issue-workspace-command-authz.test.ts": 3660,+ "server/src/__tests__/issues-checkout-wakeup.test.ts": 1053,+ "server/src/__tests__/issues-goal-context-routes.test.ts": 5356,+ "server/src/__tests__/issues-service.test.ts": 37910,+ "server/src/__tests__/llms-routes.test.ts": 2370,+ "server/src/__tests__/low-trust-red-team-routes.test.ts": 25333,+ "server/src/__tests__/multilingual-issues-routes.test.ts": 11329,+ "server/src/__tests__/onboarding-seed-route.test.ts": 8981,+ "server/src/__tests__/openapi-routes.test.ts": 3234,+ "server/src/__tests__/openclaw-invite-prompt-route.test.ts": 3938,+ "server/src/__tests__/opencode-local-adapter-environment.test.ts": 1071,+ "server/src/__tests__/permissions-upgrade-boundary-routes.test.ts": 11054,+ "server/src/__tests__/pipelines-routes.test.ts": 13514,+ "server/src/__tests__/plugin-install-route-security.test.ts": 10977,+ "server/src/__tests__/plugin-routes-authz.test.ts": 3628,+ "server/src/__tests__/plugin-scoped-api-routes.test.ts": 3535,+ "server/src/__tests__/project-goal-telemetry-routes.test.ts": 4566,+ "server/src/__tests__/project-routes-env.test.ts": 3141,+ "server/src/__tests__/projects-list-archived-routes.test.ts": 10943,+ "server/src/__tests__/redaction.test.ts": 973,+ "server/src/__tests__/resource-memberships-routes.test.ts": 11206,+ "server/src/__tests__/routine-document-annotation-routes.test.ts": 2824,+ "server/src/__tests__/routines-e2e.test.ts": 12006,+ "server/src/__tests__/routines-routes.test.ts": 4019,+ "server/src/__tests__/secret-proposals-routes.test.ts": 15670,+ "server/src/__tests__/secrets-routes.test.ts": 4757,+ "server/src/__tests__/sidebar-preferences-routes.test.ts": 3080,+ "server/src/__tests__/summary-slot-routes.test.ts": 3858,+ "server/src/__tests__/teams-catalog-routes.test.ts": 3463,+ "server/src/__tests__/user-profile-routes.test.ts": 6441,+ "server/src/__tests__/workspace-runtime-routes-authz.test.ts": 3296,+ "server/src/__tests__/workspace-runtime-service-authz.test.ts": 6251+ }+}