The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Balance CLI CI shards using measured test durations (#638)
Dee50afe8fdd280
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f3a336775..05197a790 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -202,9 +202,9 @@ jobs:# existing worker cap/isolation: subprocess fixtures still need spare CPUs.# Opt-in real-engine suites retain their existing explicit skips.- name: Discover the complete default-suite inventory- run: npx vitest list --filesOnly --exclude src/pair/learn/guard.fuzz.spec.ts --json="$RUNNER_TEMP/inventory-${{ matrix.shard }}.json"+ run: npx vitest list --config vitest.ci.config.ts --filesOnly --exclude src/pair/learn/guard.fuzz.spec.ts --json="$RUNNER_TEMP/inventory-${{ matrix.shard }}.json"- name: Run default-suite shard- run: npx vitest run --exclude src/pair/learn/guard.fuzz.spec.ts --shard=${{ matrix.shard }}/4 --reporter=default --reporter=json --outputFile.json="$RUNNER_TEMP/result-${{ matrix.shard }}.json"+ run: npx vitest run --config vitest.ci.config.ts --exclude src/pair/learn/guard.fuzz.spec.ts --shard=${{ matrix.shard }}/4 --reporter=default --reporter=json --outputFile.json="$RUNNER_TEMP/result-${{ matrix.shard }}.json"- name: Preserve inventory and test resultsif: ${{ !cancelled() }}uses: actions/upload-artifact@v4diff --git a/cli/ci-test-durations.json b/cli/ci-test-durations.jsonnew file mode 100644index 000000000..7c6c14671--- /dev/null+++ b/cli/ci-test-durations.json@@ -0,0 +1,74 @@+{+ "schema": 1,+ "sourceRun": "https://github.com/autonomous-ai/openharness/actions/runs/37050153536",+ "sourceCommit": "1488947f3e2722487bac1a56790165ff83724f2d",+ "description": "Scheduling hints only: passing CI file durations of at least 1s, rounded up to 100ms. Discovery always determines the complete test set.",+ "defaultDurationMs": 100,+ "durationMs": {+ "src/authCommand.spec.ts": 19600,+ "src/backendSocket.gridReads.spec.ts": 2900,+ "src/backendSocket.spec.ts": 11500,+ "src/cable/cableFleet.spec.ts": 2200,+ "src/cable/cableSession.spec.ts": 12900,+ "src/cliCommand.spec.ts": 16600,+ "src/config/loadEnv.spec.ts": 1500,+ "src/devicesCommand.spec.ts": 31600,+ "src/dsh/catalog.spec.ts": 1800,+ "src/dsh/command.spec.ts": 1400,+ "src/dsh/install.spec.ts": 3500,+ "src/dsh/portability.integration.spec.ts": 6600,+ "src/dsh/update.spec.ts": 1800,+ "src/dsh/verdict.spec.ts": 4700,+ "src/dsh/viewer.spec.ts": 15000,+ "src/engines/codex/portableHistory.spec.ts": 8100,+ "src/engines/cursor/discovery.spec.ts": 1200,+ "src/engines/opencode/sessionModel.spec.ts": 1500,+ "src/gridCommand.spec.ts": 17400,+ "src/hookNotify.spec.ts": 6200,+ "src/lib/askQuestion.spec.ts": 2200,+ "src/lib/autonomous-device/direct.spec.ts": 1200,+ "src/lib/autonomous-device/store.spec.ts": 1700,+ "src/lib/daemonSpawnLock.spec.ts": 2900,+ "src/lib/e2ee/deviceLogSyncer.spec.ts": 5700,+ "src/lib/e2ee/manager.test.ts": 24800,+ "src/lib/e2ee/passwordPake.spec.ts": 6400,+ "src/lib/e2ee/relayLink.spec.ts": 20800,+ "src/lib/engineInstall.user.spec.ts": 1800,+ "src/lib/engineLaunch.spec.ts": 1500,+ "src/lib/gitProject.spec.ts": 5600,+ "src/lib/gridActs.spec.ts": 6600,+ "src/lib/gridReads.spec.ts": 8500,+ "src/lib/gridTarget.spec.ts": 1500,+ "src/lib/localModels.integration.spec.ts": 2100,+ "src/lib/localModels.spec.ts": 2000,+ "src/lib/registry.spec.ts": 2600,+ "src/lib/remoteViewerRelay.spec.ts": 1100,+ "src/lib/sessionGitPullRequest.spec.ts": 1200,+ "src/lib/sessionInput.spec.ts": 2200,+ "src/lib/sessionSearch/externals/hermes.spec.ts": 1100,+ "src/lib/sessionSearch/indexer.spec.ts": 1300,+ "src/lib/stoppedAgents.spec.ts": 1300,+ "src/lib/terminalP2p.spec.ts": 6700,+ "src/lib/terminalStreamManager.spec.ts": 3000,+ "src/lib/tmux.spec.ts": 1300,+ "src/lib/tmuxStream.decode.spec.ts": 1300,+ "src/localWsServer.spec.ts": 2700,+ "src/loginForceRace.spec.ts": 3300,+ "src/machinesCommand.spec.ts": 5800,+ "src/memory/capture.spec.ts": 1500,+ "src/memory/claudeInference.spec.ts": 1200,+ "src/memory/notebook.spec.ts": 1300,+ "src/memory/opencodeCapture.spec.ts": 1800,+ "src/memory/opencodeInference.spec.ts": 1500,+ "src/memory/retention.spec.ts": 1100,+ "src/memory/runtime.spec.ts": 1100,+ "src/memory/store.spec.ts": 2000,+ "src/orchestrator/service.spec.ts": 2700,+ "src/pair/classify.fuzz.spec.ts": 2300,+ "src/pair/plateService.spec.ts": 2800,+ "src/remoteCommand.spec.ts": 1400,+ "src/scripts/install.spec.ts": 10100,+ "src/teams/remote.spec.ts": 20500,+ "src/teams/transport.spec.ts": 8200+ }+}diff --git a/cli/src/__fixtures__/ciTestSequencer.ts b/cli/src/__fixtures__/ciTestSequencer.tsnew file mode 100644index 000000000..128c18799--- /dev/null+++ b/cli/src/__fixtures__/ciTestSequencer.ts@@ -0,0 +1,60 @@+import { readFileSync } from 'node:fs'+import { relative } from 'node:path'+import { BaseSequencer, type TestSpecification } from 'vitest/node'++type TimingHints = { defaultDurationMs: number; durationMs: Record<string, number> }++export function readTimingHints(value: unknown): TimingHints {+ const raw = value as { schema?: unknown; defaultDurationMs?: unknown; durationMs?: unknown } | null+ const positive = (n: unknown): n is number => typeof n === 'number' && Number.isFinite(n) && n > 0+ if (raw?.schema !== 1 || !positive(raw.defaultDurationMs) || !raw.durationMs+ || typeof raw.durationMs !== 'object' || Array.isArray(raw.durationMs)) {+ throw new Error('Invalid CLI test timing hints')+ }+ for (const [path, duration] of Object.entries(raw.durationMs)) {+ if (!/^src\/.+\.(spec|test)\.ts$/.test(path) || path.includes('\\') || path.split('/').includes('..') || !positive(duration)) {+ throw new Error(`Invalid CLI test timing hint: ${path}`)+ }+ }+ return { defaultDurationMs: raw.defaultDurationMs, durationMs: raw.durationMs as Record<string, number> }+}++const hints = readTimingHints(JSON.parse(readFileSync(new URL('../../ci-test-durations.json', import.meta.url), 'utf8')))++/** Longest estimated files first, each assigned to the least-loaded shard. Timing hints never+ * select tests: the caller's complete discovery list is the only source of entries. New files+ * get a small default cost, and stale hints for removed files cannot add them back to the run. */+export function durationShards<T>(entries: Array<{ key: string; durationMs: number; value: T }>, count: number): T[][] {+ if (!Number.isSafeInteger(count) || count < 1) throw new Error('Shard count must be a positive integer')+ const shards = Array.from({ length: count }, () => ({ cost: 0, values: [] as T[] }))+ const keys = new Set<string>()+ for (const entry of entries) {+ if (keys.has(entry.key) || !Number.isFinite(entry.durationMs) || entry.durationMs <= 0) {+ throw new Error(`Invalid or duplicate shard entry: ${entry.key}`)+ }+ keys.add(entry.key)+ }+ const ordered = [...entries].sort((a, b) => b.durationMs - a.durationMs || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))+ for (const entry of ordered) {+ const next = shards.reduce((best, shard) => shard.cost < best.cost+ || (shard.cost === best.cost && shard.values.length < best.values.length) ? shard : best)+ next.cost += entry.durationMs+ next.values.push(entry.value)+ }+ return shards.map(shard => shard.values)+}++export class DurationSequencer extends BaseSequencer {+ override async shard(files: TestSpecification[]): Promise<TestSpecification[]> {+ const shard = this.ctx.config.shard+ if (!shard) return files+ const entries = files.map(spec => {+ const path = relative(this.ctx.config.root, spec.moduleId).replaceAll('\\', '/')+ return { key: `${spec.project.name}\0${spec.pool}\0${path}`,+ durationMs: hints.durationMs[path] ?? hints.defaultDurationMs, value: spec }+ })+ return durationShards(entries, shard.count)[shard.index - 1]+ }+ // Keep Vitest's normal failed-first/cache-aware ordering within each shard, and its+ // project/isolation/group ordering. Unsharded local runs retain the default behavior.+}diff --git a/cli/src/ciTestSequencer.spec.ts b/cli/src/ciTestSequencer.spec.tsnew file mode 100644index 000000000..fda019c61--- /dev/null+++ b/cli/src/ciTestSequencer.spec.ts@@ -0,0 +1,71 @@+import { resolve } from 'node:path'+import { describe, expect, it } from 'vitest'+import type { TestSpecification, Vitest } from 'vitest/node'+import { durationShards, DurationSequencer, readTimingHints } from './__fixtures__/ciTestSequencer.js'++const entries = (costs: number[]) => costs.map((durationMs, value) => ({ key: `file-${value}`, durationMs, value }))++describe('duration-based CLI shards', () => {+ it('spreads long files while preserving every discovered entry once', () => {+ const input = entries([300, 200, 100, 100, 100, 100, 100, 100, 100])+ const shards = durationShards(input, 4)+ expect(shards.flat().sort((a, b) => a - b)).toEqual(input.map(e => e.value))+ expect(shards.map(shard => shard.reduce((total, id) => total + input[id].durationMs, 0))).toEqual([300, 300, 300, 300])+ expect(input.map(e => e.value)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8])+ })++ it('assigns the same shards when runners discover equal-cost files in different orders', () => {+ const input = entries([100, 100, 100, 100, 100, 100, 100])+ expect(durationShards([...input].reverse(), 3)).toEqual(durationShards(input, 3))+ expect(durationShards(input, 3).map(shard => shard.length)).toEqual([3, 2, 2])+ })++ it('supports empty discovery, one shard, and more shards than files without dropping entries', () => {+ expect(durationShards([], 4)).toEqual([[], [], [], []])+ expect(durationShards(entries([10, 20]), 4).flat().sort()).toEqual([0, 1])+ expect(durationShards(entries([10, 20]), 1)[0].sort()).toEqual([0, 1])+ })++ it('rejects invalid counts, costs and ambiguous duplicate identifiers', () => {+ for (const count of [0, -1, 1.5, NaN, Infinity]) expect(() => durationShards([], count)).toThrow()+ for (const cost of [0, -1, NaN, Infinity]) expect(() => durationShards(entries([cost]), 2)).toThrow()+ expect(() => durationShards([entries([1])[0], entries([2])[0]], 2)).toThrow('duplicate')+ })++ it('rejects malformed timing data instead of silently applying a different schedule', () => {+ const valid = { schema: 1, defaultDurationMs: 100, durationMs: { 'src/slow.spec.ts': 1000 } }+ expect(readTimingHints(valid)).toEqual({ defaultDurationMs: 100, durationMs: valid.durationMs })+ for (const value of [null, {}, { ...valid, schema: 2 }, { ...valid, defaultDurationMs: 0 },+ { ...valid, durationMs: [] }, { ...valid, durationMs: { 'src/slow.spec.ts': NaN } },+ { ...valid, durationMs: { '/other/slow.spec.ts': 10 } }, { ...valid, durationMs: { 'src/../slow.spec.ts': 10 } }]) {+ expect(() => readTimingHints(value)).toThrow('timing hint')+ }+ })++ it('uses relative paths across worktrees and includes new files independently of stale hints', async () => {+ const paths = ['src/devicesCommand.spec.ts', 'src/authCommand.spec.ts', 'src/new.spec.ts', 'src/new2.spec.ts']+ const collect = async (root: string, reverse = false) => {+ const files = paths.map(path => ({ moduleId: resolve(root, path), project: { name: 'cli' }, pool: 'forks' }) as TestSpecification)+ if (reverse) files.reverse()+ return Promise.all([1, 2, 3, 4].map(index => new DurationSequencer({+ config: { root, shard: { index, count: 4 } },+ } as Vitest).shard(files).then(shard => shard.map(spec => paths.find(path => resolve(root, path) === spec.moduleId)!))))+ }+ const first = await collect(resolve('worktree-one'))+ expect(await collect(resolve('worktree-two'), true)).toEqual(first)+ expect(first.flat().sort()).toEqual([...paths].sort())+ expect(first.filter(shard => shard.includes(paths[0]) || shard.includes(paths[1]))).toHaveLength(2)+ })++ it('preserves separate project/pool specifications for the same file', async () => {+ const root = resolve('worktree')+ const files = [['first', 'forks'], ['second', 'forks'], ['second', 'typescript']].map(([name, pool]) => ({+ moduleId: resolve(root, 'src/new.spec.ts'), project: { name }, pool,+ }) as TestSpecification)+ const shards = await Promise.all([1, 2].map(index => new DurationSequencer({+ config: { root, shard: { index, count: 2 } },+ } as Vitest).shard(files)))+ expect(new Set(shards.flat())).toEqual(new Set(files))+ expect(shards.flat()).toHaveLength(files.length)+ })+})diff --git a/cli/vitest.ci.config.ts b/cli/vitest.ci.config.tsnew file mode 100644index 000000000..a27448ba0--- /dev/null+++ b/cli/vitest.ci.config.ts@@ -0,0 +1,7 @@+import { defineConfig, mergeConfig } from 'vitest/config'+import config from './vitest.config.js'+import { DurationSequencer } from './src/__fixtures__/ciTestSequencer.js'++export default mergeConfig(config, defineConfig({+ test: { sequence: { sequencer: DurationSequencer } },+}))diff --git a/docs/validation-and-release.md b/docs/validation-and-release.mdindex 13058f670..07ba6cb76 100644--- a/docs/validation-and-release.md+++ b/docs/validation-and-release.md@@ -41,6 +41,17 @@ unfinished results fail it. Review the complete workflow result, including theserial/login-shell matrix, rather than one early finishing job. Shard reports,inventories and the combined summary are retained as artifacts for seven days.+CI's `vitest.ci.config.ts` uses the slow-file timing hints in `cli/ci-test-durations.json`+to distribute estimated work across the same four runners. The complete discovered+file list remains authoritative: new files receive a default cost, and obsolete+timing entries cannot select removed files. Hints are recorded from a passing CI+run and can be refreshed from its per-file JSON durations if later runs become+uneven; stale estimates affect scheduling, never coverage. Vitest keeps its normal+ordering within each shard; the normal local configuration, including file+shuffling, is unchanged. Use `--config vitest.ci.config.ts --shard=N/4` to reproduce+the CI assignment locally. The aggregate still requires every discovered file+exactly once, independent of these estimates.+For repository process tooling only, `scope=process` runs its Python regressiontests without installing or building unrelated components. It does not validateapplication changes. Workflow edits also need `actionlint` and a run exercising