Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Balance CLI CI shards using measured test durations (#638)

Dee50afe8fdd280
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f3a336775..05197a790 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -202,9 +202,9 @@ jobs:
# existing worker cap/isolation: subprocess fixtures still need spare CPUs.
# Opt-in real-engine suites retain their existing explicit skips.
- name: Discover the complete default-suite inventory
- run: npx vitest list --filesOnly --exclude src/pair/learn/guard.fuzz.spec.ts --json="$RUNNER_TEMP/inventory-${{ matrix.shard }}.json"
+ run: npx vitest list --config vitest.ci.config.ts --filesOnly --exclude src/pair/learn/guard.fuzz.spec.ts --json="$RUNNER_TEMP/inventory-${{ matrix.shard }}.json"
- name: Run default-suite shard
- run: npx vitest run --exclude src/pair/learn/guard.fuzz.spec.ts --shard=${{ matrix.shard }}/4 --reporter=default --reporter=json --outputFile.json="$RUNNER_TEMP/result-${{ matrix.shard }}.json"
+ run: npx vitest run --config vitest.ci.config.ts --exclude src/pair/learn/guard.fuzz.spec.ts --shard=${{ matrix.shard }}/4 --reporter=default --reporter=json --outputFile.json="$RUNNER_TEMP/result-${{ matrix.shard }}.json"
- name: Preserve inventory and test results
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
diff --git a/cli/ci-test-durations.json b/cli/ci-test-durations.json
new file mode 100644
index 000000000..7c6c14671
--- /dev/null
+++ b/cli/ci-test-durations.json
@@ -0,0 +1,74 @@
+{
+ "schema": 1,
+ "sourceRun": "https://github.com/autonomous-ai/openharness/actions/runs/37050153536",
+ "sourceCommit": "1488947f3e2722487bac1a56790165ff83724f2d",
+ "description": "Scheduling hints only: passing CI file durations of at least 1s, rounded up to 100ms. Discovery always determines the complete test set.",
+ "defaultDurationMs": 100,
+ "durationMs": {
+ "src/authCommand.spec.ts": 19600,
+ "src/backendSocket.gridReads.spec.ts": 2900,
+ "src/backendSocket.spec.ts": 11500,
+ "src/cable/cableFleet.spec.ts": 2200,
+ "src/cable/cableSession.spec.ts": 12900,
+ "src/cliCommand.spec.ts": 16600,
+ "src/config/loadEnv.spec.ts": 1500,
+ "src/devicesCommand.spec.ts": 31600,
+ "src/dsh/catalog.spec.ts": 1800,
+ "src/dsh/command.spec.ts": 1400,
+ "src/dsh/install.spec.ts": 3500,
+ "src/dsh/portability.integration.spec.ts": 6600,
+ "src/dsh/update.spec.ts": 1800,
+ "src/dsh/verdict.spec.ts": 4700,
+ "src/dsh/viewer.spec.ts": 15000,
+ "src/engines/codex/portableHistory.spec.ts": 8100,
+ "src/engines/cursor/discovery.spec.ts": 1200,
+ "src/engines/opencode/sessionModel.spec.ts": 1500,
+ "src/gridCommand.spec.ts": 17400,
+ "src/hookNotify.spec.ts": 6200,
+ "src/lib/askQuestion.spec.ts": 2200,
+ "src/lib/autonomous-device/direct.spec.ts": 1200,
+ "src/lib/autonomous-device/store.spec.ts": 1700,
+ "src/lib/daemonSpawnLock.spec.ts": 2900,
+ "src/lib/e2ee/deviceLogSyncer.spec.ts": 5700,
+ "src/lib/e2ee/manager.test.ts": 24800,
+ "src/lib/e2ee/passwordPake.spec.ts": 6400,
+ "src/lib/e2ee/relayLink.spec.ts": 20800,
+ "src/lib/engineInstall.user.spec.ts": 1800,
+ "src/lib/engineLaunch.spec.ts": 1500,
+ "src/lib/gitProject.spec.ts": 5600,
+ "src/lib/gridActs.spec.ts": 6600,
+ "src/lib/gridReads.spec.ts": 8500,
+ "src/lib/gridTarget.spec.ts": 1500,
+ "src/lib/localModels.integration.spec.ts": 2100,
+ "src/lib/localModels.spec.ts": 2000,
+ "src/lib/registry.spec.ts": 2600,
+ "src/lib/remoteViewerRelay.spec.ts": 1100,
+ "src/lib/sessionGitPullRequest.spec.ts": 1200,
+ "src/lib/sessionInput.spec.ts": 2200,
+ "src/lib/sessionSearch/externals/hermes.spec.ts": 1100,
+ "src/lib/sessionSearch/indexer.spec.ts": 1300,
+ "src/lib/stoppedAgents.spec.ts": 1300,
+ "src/lib/terminalP2p.spec.ts": 6700,
+ "src/lib/terminalStreamManager.spec.ts": 3000,
+ "src/lib/tmux.spec.ts": 1300,
+ "src/lib/tmuxStream.decode.spec.ts": 1300,
+ "src/localWsServer.spec.ts": 2700,
+ "src/loginForceRace.spec.ts": 3300,
+ "src/machinesCommand.spec.ts": 5800,
+ "src/memory/capture.spec.ts": 1500,
+ "src/memory/claudeInference.spec.ts": 1200,
+ "src/memory/notebook.spec.ts": 1300,
+ "src/memory/opencodeCapture.spec.ts": 1800,
+ "src/memory/opencodeInference.spec.ts": 1500,
+ "src/memory/retention.spec.ts": 1100,
+ "src/memory/runtime.spec.ts": 1100,
+ "src/memory/store.spec.ts": 2000,
+ "src/orchestrator/service.spec.ts": 2700,
+ "src/pair/classify.fuzz.spec.ts": 2300,
+ "src/pair/plateService.spec.ts": 2800,
+ "src/remoteCommand.spec.ts": 1400,
+ "src/scripts/install.spec.ts": 10100,
+ "src/teams/remote.spec.ts": 20500,
+ "src/teams/transport.spec.ts": 8200
+ }
+}
diff --git a/cli/src/__fixtures__/ciTestSequencer.ts b/cli/src/__fixtures__/ciTestSequencer.ts
new file mode 100644
index 000000000..128c18799
--- /dev/null
+++ b/cli/src/__fixtures__/ciTestSequencer.ts
@@ -0,0 +1,60 @@
+import { readFileSync } from 'node:fs'
+import { relative } from 'node:path'
+import { BaseSequencer, type TestSpecification } from 'vitest/node'
+
+type TimingHints = { defaultDurationMs: number; durationMs: Record<string, number> }
+
+export function readTimingHints(value: unknown): TimingHints {
+ const raw = value as { schema?: unknown; defaultDurationMs?: unknown; durationMs?: unknown } | null
+ const positive = (n: unknown): n is number => typeof n === 'number' && Number.isFinite(n) && n > 0
+ if (raw?.schema !== 1 || !positive(raw.defaultDurationMs) || !raw.durationMs
+ || typeof raw.durationMs !== 'object' || Array.isArray(raw.durationMs)) {
+ throw new Error('Invalid CLI test timing hints')
+ }
+ for (const [path, duration] of Object.entries(raw.durationMs)) {
+ if (!/^src\/.+\.(spec|test)\.ts$/.test(path) || path.includes('\\') || path.split('/').includes('..') || !positive(duration)) {
+ throw new Error(`Invalid CLI test timing hint: ${path}`)
+ }
+ }
+ return { defaultDurationMs: raw.defaultDurationMs, durationMs: raw.durationMs as Record<string, number> }
+}
+
+const hints = readTimingHints(JSON.parse(readFileSync(new URL('../../ci-test-durations.json', import.meta.url), 'utf8')))
+
+/** Longest estimated files first, each assigned to the least-loaded shard. Timing hints never
+ * select tests: the caller's complete discovery list is the only source of entries. New files
+ * get a small default cost, and stale hints for removed files cannot add them back to the run. */
+export function durationShards<T>(entries: Array<{ key: string; durationMs: number; value: T }>, count: number): T[][] {
+ if (!Number.isSafeInteger(count) || count < 1) throw new Error('Shard count must be a positive integer')
+ const shards = Array.from({ length: count }, () => ({ cost: 0, values: [] as T[] }))
+ const keys = new Set<string>()
+ for (const entry of entries) {
+ if (keys.has(entry.key) || !Number.isFinite(entry.durationMs) || entry.durationMs <= 0) {
+ throw new Error(`Invalid or duplicate shard entry: ${entry.key}`)
+ }
+ keys.add(entry.key)
+ }
+ const ordered = [...entries].sort((a, b) => b.durationMs - a.durationMs || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
+ for (const entry of ordered) {
+ const next = shards.reduce((best, shard) => shard.cost < best.cost
+ || (shard.cost === best.cost && shard.values.length < best.values.length) ? shard : best)
+ next.cost += entry.durationMs
+ next.values.push(entry.value)
+ }
+ return shards.map(shard => shard.values)
+}
+
+export class DurationSequencer extends BaseSequencer {
+ override async shard(files: TestSpecification[]): Promise<TestSpecification[]> {
+ const shard = this.ctx.config.shard
+ if (!shard) return files
+ const entries = files.map(spec => {
+ const path = relative(this.ctx.config.root, spec.moduleId).replaceAll('\\', '/')
+ return { key: `${spec.project.name}\0${spec.pool}\0${path}`,
+ durationMs: hints.durationMs[path] ?? hints.defaultDurationMs, value: spec }
+ })
+ return durationShards(entries, shard.count)[shard.index - 1]
+ }
+ // Keep Vitest's normal failed-first/cache-aware ordering within each shard, and its
+ // project/isolation/group ordering. Unsharded local runs retain the default behavior.
+}
diff --git a/cli/src/ciTestSequencer.spec.ts b/cli/src/ciTestSequencer.spec.ts
new file mode 100644
index 000000000..fda019c61
--- /dev/null
+++ b/cli/src/ciTestSequencer.spec.ts
@@ -0,0 +1,71 @@
+import { resolve } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import type { TestSpecification, Vitest } from 'vitest/node'
+import { durationShards, DurationSequencer, readTimingHints } from './__fixtures__/ciTestSequencer.js'
+
+const entries = (costs: number[]) => costs.map((durationMs, value) => ({ key: `file-${value}`, durationMs, value }))
+
+describe('duration-based CLI shards', () => {
+ it('spreads long files while preserving every discovered entry once', () => {
+ const input = entries([300, 200, 100, 100, 100, 100, 100, 100, 100])
+ const shards = durationShards(input, 4)
+ expect(shards.flat().sort((a, b) => a - b)).toEqual(input.map(e => e.value))
+ expect(shards.map(shard => shard.reduce((total, id) => total + input[id].durationMs, 0))).toEqual([300, 300, 300, 300])
+ expect(input.map(e => e.value)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8])
+ })
+
+ it('assigns the same shards when runners discover equal-cost files in different orders', () => {
+ const input = entries([100, 100, 100, 100, 100, 100, 100])
+ expect(durationShards([...input].reverse(), 3)).toEqual(durationShards(input, 3))
+ expect(durationShards(input, 3).map(shard => shard.length)).toEqual([3, 2, 2])
+ })
+
+ it('supports empty discovery, one shard, and more shards than files without dropping entries', () => {
+ expect(durationShards([], 4)).toEqual([[], [], [], []])
+ expect(durationShards(entries([10, 20]), 4).flat().sort()).toEqual([0, 1])
+ expect(durationShards(entries([10, 20]), 1)[0].sort()).toEqual([0, 1])
+ })
+
+ it('rejects invalid counts, costs and ambiguous duplicate identifiers', () => {
+ for (const count of [0, -1, 1.5, NaN, Infinity]) expect(() => durationShards([], count)).toThrow()
+ for (const cost of [0, -1, NaN, Infinity]) expect(() => durationShards(entries([cost]), 2)).toThrow()
+ expect(() => durationShards([entries([1])[0], entries([2])[0]], 2)).toThrow('duplicate')
+ })
+
+ it('rejects malformed timing data instead of silently applying a different schedule', () => {
+ const valid = { schema: 1, defaultDurationMs: 100, durationMs: { 'src/slow.spec.ts': 1000 } }
+ expect(readTimingHints(valid)).toEqual({ defaultDurationMs: 100, durationMs: valid.durationMs })
+ for (const value of [null, {}, { ...valid, schema: 2 }, { ...valid, defaultDurationMs: 0 },
+ { ...valid, durationMs: [] }, { ...valid, durationMs: { 'src/slow.spec.ts': NaN } },
+ { ...valid, durationMs: { '/other/slow.spec.ts': 10 } }, { ...valid, durationMs: { 'src/../slow.spec.ts': 10 } }]) {
+ expect(() => readTimingHints(value)).toThrow('timing hint')
+ }
+ })
+
+ it('uses relative paths across worktrees and includes new files independently of stale hints', async () => {
+ const paths = ['src/devicesCommand.spec.ts', 'src/authCommand.spec.ts', 'src/new.spec.ts', 'src/new2.spec.ts']
+ const collect = async (root: string, reverse = false) => {
+ const files = paths.map(path => ({ moduleId: resolve(root, path), project: { name: 'cli' }, pool: 'forks' }) as TestSpecification)
+ if (reverse) files.reverse()
+ return Promise.all([1, 2, 3, 4].map(index => new DurationSequencer({
+ config: { root, shard: { index, count: 4 } },
+ } as Vitest).shard(files).then(shard => shard.map(spec => paths.find(path => resolve(root, path) === spec.moduleId)!))))
+ }
+ const first = await collect(resolve('worktree-one'))
+ expect(await collect(resolve('worktree-two'), true)).toEqual(first)
+ expect(first.flat().sort()).toEqual([...paths].sort())
+ expect(first.filter(shard => shard.includes(paths[0]) || shard.includes(paths[1]))).toHaveLength(2)
+ })
+
+ it('preserves separate project/pool specifications for the same file', async () => {
+ const root = resolve('worktree')
+ const files = [['first', 'forks'], ['second', 'forks'], ['second', 'typescript']].map(([name, pool]) => ({
+ moduleId: resolve(root, 'src/new.spec.ts'), project: { name }, pool,
+ }) as TestSpecification)
+ const shards = await Promise.all([1, 2].map(index => new DurationSequencer({
+ config: { root, shard: { index, count: 2 } },
+ } as Vitest).shard(files)))
+ expect(new Set(shards.flat())).toEqual(new Set(files))
+ expect(shards.flat()).toHaveLength(files.length)
+ })
+})
diff --git a/cli/vitest.ci.config.ts b/cli/vitest.ci.config.ts
new file mode 100644
index 000000000..a27448ba0
--- /dev/null
+++ b/cli/vitest.ci.config.ts
@@ -0,0 +1,7 @@
+import { defineConfig, mergeConfig } from 'vitest/config'
+import config from './vitest.config.js'
+import { DurationSequencer } from './src/__fixtures__/ciTestSequencer.js'
+
+export default mergeConfig(config, defineConfig({
+ test: { sequence: { sequencer: DurationSequencer } },
+}))
diff --git a/docs/validation-and-release.md b/docs/validation-and-release.md
index 13058f670..07ba6cb76 100644
--- a/docs/validation-and-release.md
+++ b/docs/validation-and-release.md
@@ -41,6 +41,17 @@ unfinished results fail it. Review the complete workflow result, including the
serial/login-shell matrix, rather than one early finishing job. Shard reports,
inventories and the combined summary are retained as artifacts for seven days.
+CI's `vitest.ci.config.ts` uses the slow-file timing hints in `cli/ci-test-durations.json`
+to distribute estimated work across the same four runners. The complete discovered
+file list remains authoritative: new files receive a default cost, and obsolete
+timing entries cannot select removed files. Hints are recorded from a passing CI
+run and can be refreshed from its per-file JSON durations if later runs become
+uneven; stale estimates affect scheduling, never coverage. Vitest keeps its normal
+ordering within each shard; the normal local configuration, including file
+shuffling, is unchanged. Use `--config vitest.ci.config.ts --shard=N/4` to reproduce
+the CI assignment locally. The aggregate still requires every discovered file
+exactly once, independent of these estimates.
+
For repository process tooling only, `scope=process` runs its Python regression
tests without installing or building unrelated components. It does not validate
application changes. Workflow edits also need `actionlint` and a run exercising