Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
ci: crash robot runs two at a time; split reconcile-owner-journey by transport; pregate false positives
garrytane603ec37566c
diff --git a/.gstack/pregate.json b/.gstack/pregate.jsonindex 5ac2ec0970..a1a153591a 100644--- a/.gstack/pregate.json+++ b/.gstack/pregate.json@@ -117,5 +117,14 @@],"visibility": "public"},- "remote_gate": "bash scripts/pregate-remote-gate.sh"+ "remote_gate": "bash scripts/pregate-remote-gate.sh",+ "allowlist": [+ {+ "finding": "pii.phone.e164",+ "paths": [+ "scripts/*-weights.metadata.json"+ ],+ "reason": "shard-weight metadata records GitHub Actions run ids and commit counts, never phone numbers"+ }+ ]}diff --git a/docs/ci-event-parity.md b/docs/ci-event-parity.mdindex c284c59760..36357774b4 100644--- a/docs/ci-event-parity.md+++ b/docs/ci-event-parity.md@@ -174,6 +174,27 @@ Measured on ubicloud-standard-4, full scope, run 39996381819:`overlapping_slugs_two_sources` (2.4 s in the other cell). 880 s is the slowest cell plusabout 30%; the nightly's two exploration seeds (about 15–20 s each on Postgres) fit insidethat margin.+- **Two runs at a time (#8036).** A full Postgres run went 899 s against 880 s on a+ pull request that doesn't touch persistence (run 41087538943). Across the 13 full-scope Postgres+ cells on ubicloud-standard-4 from 2026-10-30, wall time ranged from 379 s to 899 s with the same+ 54 runs. Two things varied: runner speed (the runs other than the two waiting faults took+ 342 s to 769 s), and whether `pooler_disconnect` waited out a 2-minute effect-claim lease+ (125–132 s in 8 of the 13 cells, 2–8 s in the rest). One run at a time adds both waits to the wall+ clock in full. Since #8036 the driver runs `ROBOT_CONCURRENCY` (2) runs at a time. Each run+ still owns its database, scratch root and home. Every fault is drawn before any crash runs, so+ the runs, the faults and the manifest order are the same as one at a time (unit-tested, and+ identical in all four local manifests below). The process faults start first, so their waits+ overlap the crash runs.++ | Measurement | One at a time | Two at a time |+ |---|---|---|+ | Local postgres, 4 vCPU, full scope | 423 s | 258 s (each run about 15% slower) |+ | Local pglite, 4 vCPU, full scope | 699 s | 418 s (each run about 20% slower) |+ | Slowest CI cell (run 41087538943) | 899 s measured | 528 s predicted |++ The prediction replays that cell's 54 measured run times, slowed by 15%, through two lanes. The+ same model predicts 250 s for the local cell, which measured 258 s. The ceilings are unchanged,+ so the slowest cell observed now uses about 60% of 880 s.- **Pull requests, 820 s PGLite and 620 s Postgres.** Scope `pr` crashes each seam onceand skips the Postgres lease-bound seams. PGLite measured 510 s and 427 s (run40003479271), then 529 s and 474 s (run 40018836178); 820 s is the slowest plus about 30%.diff --git a/scripts/persistence/README.md b/scripts/persistence/README.mdindex dbcda05b3a..ece6cc1226 100644--- a/scripts/persistence/README.md+++ b/scripts/persistence/README.md@@ -316,9 +316,11 @@ revision and disclose skipped cells.on PGLite and, with an explicit safe `DATABASE_URL`, isolated PostgreSQL databases.It covers stale preconditions, current/original grants, private facts, retainedbackups, ordinary mutations after repair, and competing publications.-`test/reconcile-owner-journey.serial.test.ts` drives real CLI requests through-HTTP and stdio PGLite owners before and after activation, restarts the owner, and-independently reads the newly remembered private fact and provenance.+`test/reconcile-owner-journey.serial.test.ts` (stdio) and+`test/reconcile-owner-journey-http.serial.test.ts` (HTTP) drive real CLI requests+through PGLite owners before and after activation, restart the owner, and+independently read the newly remembered private fact and provenance+(`test/helpers/reconcile-owner-journey.ts`).`test/reconcile-crash-*.slow.test.ts` and `test/e2e/reconcile-crash*.test.ts` kill realprocesses at all eight publication boundaries with activation off/on. PostgreSQLdiff --git a/scripts/persistence/robot-driver.ts b/scripts/persistence/robot-driver.tsindex ca09325cd6..d2453d337a 100644--- a/scripts/persistence/robot-driver.ts+++ b/scripts/persistence/robot-driver.ts@@ -10,7 +10,7 @@*/import { createHash, randomUUID } from 'node:crypto';import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';-import { join, resolve } from 'node:path';+import { basename, join, resolve } from 'node:path';import type postgres from '#postgres';import { random } from './harness.ts';import { crossBoundarySequences, randomSchedule, type GeneratorTopology, type Schedule } from './generator.ts';@@ -41,6 +41,12 @@ export type RobotScope = 'pr' | 'full';* fence_absent fixtures of #7435.*/export const ROBOT_RANDOM_SEEDS: readonly number[] = [5105, 5106, 5107, 5108, 5109, 5113, 5115, 5116];+/**+ * Runs in flight at once. Every run owns its database, scratch root and home, and the plan is fixed before any+ * crash runs, so the width changes only the wall clock (#8036). Two is measured on ubicloud-standard-4+ * (docs/ci-event-parity.md#crash-robot-ceilings).+ */+export const ROBOT_CONCURRENCY = 2;/** Seeds from 5109 to 5116 not in ROBOT_RANDOM_SEEDS: the nightly runs one of them each night. */export const ROBOT_ROTATION_SEEDS: readonly number[] = [5110, 5111, 5112, 5114];/** The nightly's exploration seeds for a UTC day number: one rotation seed (each within four nights) and one never run before. */@@ -92,6 +98,8 @@ export interface RobotOptions {/** Keep only these op ids of a schedule (shrinking). */keep?: Map<string, Set<string>>;worktrees?: number;+ /** Runs in flight at once (default ROBOT_CONCURRENCY); a replay runs one at a time. */+ concurrency?: number;log?: (line: string) => void;}@@ -143,9 +151,10 @@ export async function runRobotPhase(o: RobotOptions) {const { database, ...place } = await fresh(`${schedule.label}-${role}`);const config: RobotConfig = { kind: o.engine, ...place, schedule, worktrees: o.worktrees ?? 2, statePath: join(place.root, 'state.json'), ...extra };const path = join(place.root, 'robot.json'); writeFileSync(path, JSON.stringify(config), { mode: 0o600 });+ const home = join(o.home, basename(place.root)); mkdirSync(home, { recursive: true });const base = { schedule: schedule.label, seed: schedule.seed, length: schedule.ops.length, ...(extra.fault ? { fault: extra.fault } : {}),...(extra.process ? { process: extra.process } : {}), ...(o.keep?.has(schedule.label) ? { ops: schedule.ops.map(d => d.id) } : {}) };- const child = o.spawn(path, o.home, 'robot', [role], environment); o.track(child);+ const child = o.spawn(path, home, 'robot', [role], environment); o.track(child);const outcome = async (): Promise<RobotRun & { counts?: Record<string, number> }> => {if (role === 'run' && extra.fault) {const reached = await Promise.race([child.event('fault', 600_000).then(e => ({ fault: e })),@@ -159,7 +168,7 @@ export async function runRobotPhase(o: RobotOptions) {return { ...base, crashed: false, fault_not_reached: true, violations: result.violations, deferred: result.deferred, trace: result.trace, duration_ms: performance.now() - at };}await child.kill();- const recovered = o.spawn(path, o.home, 'robot', ['recover'], environment); o.track(recovered);+ const recovered = o.spawn(path, home, 'robot', ['recover'], environment); o.track(recovered);const result = (await recovered.done()).result as RobotOutcome;return { ...base, crashed: true, violations: result.violations, deferred: result.deferred, trace: result.trace, duration_ms: performance.now() - at };}@@ -170,14 +179,24 @@ export async function runRobotPhase(o: RobotOptions) {if (!runFailed(run)) await release(database);return run;}- const record = (run: RobotRun) => {- runs.push(run);+ const say = (run: RobotRun) => {const what = run.fault ? `${run.fault.point}#${run.fault.nth}` : run.process ?? 'no crash';const verdict = run.violations.length ? `${run.violations.length} VIOLATION(S) ${run.violations.map(v => v.class).join(',')}`: run.hook_error !== undefined ? `FAULT HOOK ERROR ${run.hook_error}`: run.fault_not_reached ? (unreachedAllowed(run) ? 'fault not reached (allowed)' : 'FAULT NOT REACHED') : 'ok';log(`${run.schedule} ${what}: ${verdict} (${Math.round(run.duration_ms)} ms)`);};+ const record = (run: RobotRun) => { runs.push(run); say(run); };+ const go = (schedule: Schedule, role: 'count' | 'run', extra: Partial<RobotConfig> = {}) => execute(schedule, role, extra).then(run => { say(run); return run; });+ const width = o.concurrency ?? ROBOT_CONCURRENCY;+ /** `each` over `items`, `width` at a time, each result at its item's index. */+ async function pooled<T, R>(items: readonly T[], each: (item: T) => Promise<R>): Promise<R[]> {+ const results: R[] = [];+ let next = 0;+ const lane = async () => { while (next < items.length) { const index = next++; results[index] = await each(items[index]!); } };+ await Promise.all(Array.from({ length: Math.min(width, items.length) }, lane));+ return results;+ }if (o.replay) {for (const entry of o.replay) {@@ -189,48 +208,59 @@ export async function runRobotPhase(o: RobotOptions) {const rand = random(o.seed);const scope = o.scope ?? 'full';+ const fixed = crossBoundarySequences(ROBOT_TOPOLOGY, o.seed);+ const seeded = [...ROBOT_RANDOM_SEEDS, ...(o.extraSeeds ?? [])].map(seed => randomSchedule(ROBOT_TOPOLOGY, seed >>> 0, 24));+ // Every schedule is counted first and every fault is then drawn in one fixed order, so how many runs are in+ // flight never changes which runs execute, and the manifest lists them in that order.+ const counted = await pooled([...fixed, ...seeded], schedule => go(schedule, 'count'));+ const fixedCounts = counted.slice(0, fixed.length);+ const seededCounts = counted.slice(fixed.length);// Every cross-boundary sequence runs uncrashed, then each seam it reaches is crashed once,// seams first covered across sequences (a seam no run has crashed yet goes first).- const fixed = crossBoundarySequences(ROBOT_TOPOLOGY, o.seed);const plan: { schedule: Schedule; point: FaultPoint; count: number }[] = [];const skipLease = o.engine === 'postgres' && scope === 'pr';// A full PGLite run crashes every seam of every sequence; Postgres and pull requests crash each seam once.const everySeam = scope === 'full' && o.engine === 'pglite';const skipped = new Set<string>();- for (const schedule of fixed) {- const counted = await execute(schedule, 'count'); record(counted);- for (const [point, count] of Object.entries(counted.counts ?? {}) as [FaultPoint, number][]) {+ fixed.forEach((schedule, index) => {+ for (const [point, count] of Object.entries(fixedCounts[index]!.counts ?? {}) as [FaultPoint, number][]) {if (skipLease && leaseBoundSeam(point)) { skipped.add(point); continue; }plan.push({ schedule, point, count });}- }+ });// The seed rotates which sequence covers each seam first.const turn = (x: (typeof plan)[number]) => (fixed.indexOf(x.schedule) + o.seed) % fixed.length;plan.sort((x, y) => x.point.localeCompare(y.point) || turn(x) - turn(y));const firsts = plan.filter((x, i) => plan.findIndex(y => y.point === x.point) === i);const ordered = [...firsts, ...plan.filter(x => !firsts.includes(x))];- for (const item of everySeam ? ordered : firsts) {- record(await execute(item.schedule, 'run', { fault: { point: item.point, nth: 1 + Math.floor(rand() * item.count) } }));- }+ type Job = { schedule: Schedule; extra: Partial<RobotConfig> };+ const crashes: Job[] = (everySeam ? ordered : firsts)+ .map(item => ({ schedule: item.schedule, extra: { fault: { point: item.point, nth: 1 + Math.floor(rand() * item.count) } } }));const skippedFaults = new Set<ProcessFault>();+ const processes: Job[] = [];for (const fault of PROCESS_FAULTS) {if (fault === 'pooler_disconnect' && !o.admin) continue;if (skipLease && LEASE_BOUND_FAULTS.includes(fault)) { skippedFaults.add(fault); continue; }- record(await execute(fixed[1], 'run', { process: fault }));+ processes.push({ schedule: fixed[1]!, extra: { process: fault } });}// serve killed right after the facts-absorb job's extraction commits; the job then runs again.- record(await execute({ ...fixed[1], ops: [], groups: [] }, 'run', { process: 'facts_absorb_kill',- fault: { point: 'publication:after_commit', nth: 1, operation: 'extract_facts' } }));+ processes.push({ schedule: { ...fixed[1]!, ops: [], groups: [] }, extra: { process: 'facts_absorb_kill',+ fault: { point: 'publication:after_commit', nth: 1, operation: 'extract_facts' } } });// The fixed random sequences, each counted and crashed once. On Postgres the crash avoids the lease-bound// seams: the cross-boundary sequences already crash each of them once.- for (const seed of [...ROBOT_RANDOM_SEEDS, ...(o.extraSeeds ?? [])]) {- const schedule = randomSchedule(ROBOT_TOPOLOGY, seed >>> 0, 24);- const counted = await execute(schedule, 'count'); record(counted);- const points = (Object.entries(counted.counts ?? {}) as [FaultPoint, number][]).filter(([point]) => !(o.engine === 'postgres' && leaseBoundSeam(point)));- if (!points.length) continue;- const [point, count] = points[Math.floor(rand() * points.length)];- record(await execute(schedule, 'run', { fault: { point, nth: 1 + Math.floor(rand() * count) } }));- }+ const seededCrashes = seeded.map((schedule, index): Job | undefined => {+ const points = (Object.entries(seededCounts[index]!.counts ?? {}) as [FaultPoint, number][]).filter(([point]) => !(o.engine === 'postgres' && leaseBoundSeam(point)));+ if (!points.length) return undefined;+ const [point, count] = points[Math.floor(rand() * points.length)]!;+ return { schedule, extra: { fault: { point, nth: 1 + Math.floor(rand() * count) } } };+ });+ // The process faults start first: a hung git child and a dead owner's claim lease mostly wait, so they+ // overlap the crash runs instead of adding their wait to the wall clock.+ const jobs = [...processes, ...crashes, ...seededCrashes.filter((job): job is Job => job !== undefined)];+ const finished = await pooled(jobs, job => go(job.schedule, 'run', job.extra));+ const result = (job: Job) => finished[jobs.indexOf(job)]!;+ runs.push(...fixedCounts, ...crashes.map(result), ...processes.map(result));+ seeded.forEach((_, index) => { runs.push(seededCounts[index]!); const job = seededCrashes[index]; if (job) runs.push(result(job)); });const summary = summarize(runs, started, o);return { ...summary, scope, over_budget: o.seconds > 0 && summary.duration_ms > o.seconds * 1000,lease_bound_seams_skipped: [...skipped].sort(), lease_bound_faults_skipped: [...skippedFaults].sort() };diff --git a/scripts/serial-files.tsv b/scripts/serial-files.tsvindex 3d5b157760..b552aec386 100644--- a/scripts/serial-files.tsv+++ b/scripts/serial-files.tsv@@ -334,6 +334,7 @@ test/query-op-limit-mode-4356.serial.test.ts R2 R2: mock.module(../src/core/seartest/readiness-embedding-enablement.serial.test.ts global-state global-state: R3: constructs PGLiteEngine outside beforeAlltest/recall-budget-policy-thin.serial.test.ts R2 R2: mock.module(../src/core/config.ts, ../src/core/mcp-client.ts) replaces modules for every file in the processtest/recall-thin-client-fallback.serial.test.ts R2 R2: mock.module(../src/core/config.ts, ../src/core/mcp-client.ts) replaces modules for every file in the process+test/reconcile-owner-journey-http.serial.test.ts global-state global-state: R3: constructs PGLiteEngine outside beforeAll; R4: PGLiteEngine without an afterAll disconnect; binds local servers/portstest/reconcile-owner-journey.serial.test.ts global-state global-state: R3: constructs PGLiteEngine outside beforeAll; R4: PGLiteEngine without an afterAll disconnect; binds local servers/portstest/reindex-code-max-cost.serial.test.ts global-state global-state: configureGateway changes the process-wide AI gatewaytest/reindex-code-model-source.serial.test.ts R1 R1: mutates process.env (GBRAIN_NO_CODE_MODEL_NUDGE)diff --git a/scripts/test-shard.sh b/scripts/test-shard.shindex 2230b93c0c..f6979d5465 100755--- a/scripts/test-shard.sh+++ b/scripts/test-shard.sh@@ -161,7 +161,7 @@ receipt_begin primary "s${SHARD_INDEX}of${TOTAL_SHARDS}" "$SHARD_INDEX" "$TOTAL_[ "${#RECEIPT_ARGS[@]}" -eq 0 ] || XARGS_FLAGS=(-n 100000 -x)# --max-concurrency mirrors the local runner: unbounded intra-process# concurrency under parallel PGLite boots produced real shard deaths (the-# 22-minute matrix timeout in test.yml records 13 of them).+# matrix job's timeout-minutes in test.yml records 13 of them).## The `./` prefix makes each argument a path, not a name filter: Bun runs# bare `test/x.test.ts` arguments in directory-scan order (filesystemdiff --git a/test/helpers/reconcile-owner-journey.ts b/test/helpers/reconcile-owner-journey.tsindex e4a2db7d75..26db4d93fa 100644--- a/test/helpers/reconcile-owner-journey.ts+++ b/test/helpers/reconcile-owner-journey.ts@@ -1,22 +1,22 @@import { expect, test } from 'bun:test';-import { PENDING_WRITE_EXIT_CODE } from '../src/core/exit-codes.ts';+import { PENDING_WRITE_EXIT_CODE } from '../../src/core/exit-codes.ts';import { randomUUID } from 'node:crypto';import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';-import { PGLiteEngine } from '../src/core/pglite-engine.ts';-import { importFromContent } from '../src/core/import-file.ts';-import { parseMarkdown, serializePageToMarkdown } from '../src/core/markdown.ts';-import { parseFactsFence } from '../src/core/facts-fence.ts';-import { inspectLockHolder } from '../src/core/pglite-lock.ts';-import { claimWorktree, getWorktreeBinding } from '../src/core/persistence/ownership.ts';-import { registerLocalWriter } from '../src/core/persistence/identity.ts';-import { acquireNativeLock, type NativeLockHandle } from '../src/core/persistence/native-lock.ts';-import { persistenceSocketPathForConfig, requestPersistenceCapabilities } from '../src/core/persistence/ipc.ts';-import { runCli, type CliResult } from './helpers/cli-spawn.ts';-import { keylessBrainEnv, PROVIDER_ENV_KEYS } from './helpers/provider-env.ts';-import { withEnv } from './helpers/with-env.ts';-import { waitFor } from './helpers/wait-for.ts';+import { PGLiteEngine } from '../../src/core/pglite-engine.ts';+import { importFromContent } from '../../src/core/import-file.ts';+import { parseMarkdown, serializePageToMarkdown } from '../../src/core/markdown.ts';+import { parseFactsFence } from '../../src/core/facts-fence.ts';+import { inspectLockHolder } from '../../src/core/pglite-lock.ts';+import { claimWorktree, getWorktreeBinding } from '../../src/core/persistence/ownership.ts';+import { registerLocalWriter } from '../../src/core/persistence/identity.ts';+import { acquireNativeLock, type NativeLockHandle } from '../../src/core/persistence/native-lock.ts';+import { persistenceSocketPathForConfig, requestPersistenceCapabilities } from '../../src/core/persistence/ipc.ts';+import { runCli, type CliResult } from './cli-spawn.ts';+import { keylessBrainEnv, PROVIDER_ENV_KEYS } from './provider-env.ts';+import { withEnv } from './with-env.ts';+import { waitFor } from './wait-for.ts';const slug = 'people/example';const sourceId = 'workspace';@@ -49,7 +49,7 @@ function privateSummary(result: CliResult) {expect(result.stdout).not.toContain('compiled_truth');}-for (const transport of ['stdio', 'http'] as const) {+export function ownerJourney(transport: 'stdio' | 'http') {for (const enabled of [false, true]) {test(`${transport} resident owner: preview, resolve, pending/restart/replay, and independent remember readback with activation ${enabled ? 'on' : 'off'}`, async () => {const home = mkdtempSync(join(tmpdir(), 'gbrain-reconcile-owner-'));@@ -99,8 +99,8 @@ for (const transport of ['stdio', 'http'] as const) {const port = reservation?.port;await reservation?.stop(true);const args = transport === 'stdio'- ? [join(import.meta.dir, 'fixtures/persistence-stdio-owner.ts')]- : [join(import.meta.dir, '../src/cli.ts'), 'serve', '--http', '--bind', '127.0.0.1', '--port', String(port)];+ ? [join(import.meta.dir, '../fixtures/persistence-stdio-owner.ts')]+ : [join(import.meta.dir, '../../src/cli.ts'), 'serve', '--http', '--bind', '127.0.0.1', '--port', String(port)];owner = Bun.spawn([process.execPath, '--no-env-file', ...args], {cwd: home, env, stdin: 'pipe', stdout: 'pipe', stderr: 'pipe',});diff --git a/test/persistence-crash-robot.test.ts b/test/persistence-crash-robot.test.tsindex 494e71aac8..1a2367a232 100644--- a/test/persistence-crash-robot.test.ts+++ b/test/persistence-crash-robot.test.ts@@ -20,7 +20,7 @@ import { join } from 'node:path';import { descriptor, executeOp, parseDescriptor, SessionDrops } from '../scripts/persistence/ops.ts';import { ReferenceModel } from '../scripts/persistence/model.ts';import { crossBoundarySequences, pageBody, randomSchedule } from '../scripts/persistence/generator.ts';-import { ROBOT_RANDOM_SEEDS, ROBOT_ROTATION_SEEDS, ROBOT_TOPOLOGY, explorationSeeds, restrict, runRobotPhase, type RobotRun } from '../scripts/persistence/robot-driver.ts';+import { ROBOT_CONCURRENCY, ROBOT_RANDOM_SEEDS, ROBOT_ROTATION_SEEDS, ROBOT_TOPOLOGY, explorationSeeds, restrict, runRobotPhase, type RobotRun } from '../scripts/persistence/robot-driver.ts';import { robotCeilingUse, robotFailure, robotFullGate, robotOverBudget, runValidation, spawnWorker } from '../scripts/persistence/validate.ts';import { faultPoint, installFaultHook, type FaultDetail, type FaultPoint } from '../src/core/persistence/fault-points.ts';import { shrinkRun } from '../scripts/persistence/shrink.ts';@@ -581,21 +581,28 @@ await faultPoint('effect:git:mid', { requestId: 'r1', signal: cyclic as never })describe('every robot run covers a fixed schedule list (reuse of a passed run is sound)', () => {const COUNTS = { 'consumer:claiming': 2, 'effect:git:mid': 1 };const clean = { violations: [], deferred: [] };- async function plan(o: { engine?: 'pglite' | 'postgres'; seconds: number; scope?: 'pr' | 'full'; extraSeeds?: number[] }) {+ async function plan(o: { engine?: 'pglite' | 'postgres'; seconds: number; scope?: 'pr' | 'full'; extraSeeds?: number[]; concurrency?: number; workerMs?: number }) {const scratch = mkdtempSync(join(tmpdir(), 'gbrain-crash-robot-plan-'));const calls: { schedule: string; role: string; point?: string; process?: string }[] = [];+ let inFlight = 0; let widest = 0;const spawn = ((path: string, _home: string, _mode: string, [role]: string[]) => {const config = JSON.parse(readFileSync(path, 'utf8'));calls.push({ schedule: config.schedule.label, role, ...(config.fault ? { point: config.fault.point } : {}), ...(config.process ? { process: config.process } : {}) });+ if (role !== 'recover') widest = Math.max(widest, ++inFlight);const events: Record<string, unknown> = role === 'count' ? { done: { result: { ...clean, counts: COUNTS } } }: role === 'run' && config.fault ? { fault: config.fault } : { done: { result: clean } };- const event = (name: string) => name in events ? Promise.resolve({ event: name, ...(events[name] as object) }) : new Promise<never>(() => {});+ const event = async (name: string) => {+ if (!(name in events)) return new Promise<never>(() => {});+ await Bun.sleep(o.workerMs ?? 0);+ if (name === 'done' && role !== 'recover' || name === 'fault') inFlight--;+ return { event: name, ...(events[name] as object) };+ };return { child: { pid: 0 }, event, done: () => event('done'), failureDiagnostics: () => [], async kill() {} };}) as unknown as typeof spawnWorker;try {const robot = await runRobotPhase({ engine: o.engine ?? 'pglite', seed: 0, seconds: o.seconds, scope: o.scope, extraSeeds: o.extraSeeds,- scratch, home: scratch, databases: [], spawn, track: () => {}, log: () => {} });- return { robot, calls };+ concurrency: o.concurrency, scratch, home: scratch, databases: [], spawn, track: () => {}, log: () => {} });+ return { robot, calls, widest };} finally { rmSync(scratch, { recursive: true, force: true }); }}const randomLabels = (calls: { schedule: string; role: string }[]) => [...new Set(calls.filter(c => c.schedule.startsWith('random-')).map(c => c.schedule))];@@ -623,6 +630,17 @@ describe('every robot run covers a fixed schedule list (reuse of a passed run isexpect(((await plan({ engine: 'postgres', seconds: 1, scope: 'pr' })).robot as { lease_bound_seams_skipped?: string[] }).lease_bound_seams_skipped).toEqual(['effect:git:mid']);});+ test('runs execute ROBOT_CONCURRENCY at a time and cover exactly the runs, faults and manifest order of one at a time (#8036)', async () => {+ const key = (r: RobotRun) => JSON.stringify([r.schedule, r.seed, r.fault ?? null, r.process ?? null]);+ for (const o of [{ engine: 'pglite' as const }, { engine: 'postgres' as const }, { engine: 'postgres' as const, scope: 'pr' as const }]) {+ const serial = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), concurrency: 1, workerMs: 1 });+ const parallel = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), workerMs: 1 });+ expect({ serial: serial.widest, parallel: parallel.widest }).toEqual({ serial: 1, parallel: ROBOT_CONCURRENCY });+ expect(parallel.robot.runs.map(key)).toEqual(serial.robot.runs.map(key));+ expect([...parallel.calls].map(c => JSON.stringify(c)).sort()).toEqual([...serial.calls].map(c => JSON.stringify(c)).sort());+ }+ });+test('a run past its ceiling is over_budget, and validation fails it loudly', async () => {expect((await plan({ seconds: 0.001 })).robot).toMatchObject({ over_budget: true });expect((await plan({ seconds: 600 })).robot).toMatchObject({ over_budget: false });diff --git a/test/reconcile-owner-journey-http.serial.test.ts b/test/reconcile-owner-journey-http.serial.test.tsnew file mode 100644index 0000000000..3d1464ca2e--- /dev/null+++ b/test/reconcile-owner-journey-http.serial.test.ts@@ -0,0 +1,3 @@+import { ownerJourney } from './helpers/reconcile-owner-journey.ts';++ownerJourney('http');diff --git a/test/reconcile-owner-journey.serial.test.ts b/test/reconcile-owner-journey.serial.test.tsnew file mode 100644index 0000000000..a43cfa6b2d--- /dev/null+++ b/test/reconcile-owner-journey.serial.test.ts@@ -0,0 +1,3 @@+import { ownerJourney } from './helpers/reconcile-owner-journey.ts';++ownerJourney('stdio');diff --git a/test/scripts/unit-shard-budget.test.ts b/test/scripts/unit-shard-budget.test.tsindex 4d52f75193..d486b2123b 100644--- a/test/scripts/unit-shard-budget.test.ts+++ b/test/scripts/unit-shard-budget.test.ts@@ -1,5 +1,5 @@/**- * test.yml's unit shards run inside the `test` job's 22-minute timeout. A+ * test.yml's unit shards run inside the `test` job's timeout-minutes. A* shard's mined weight must fit that timeout twice over, after setup, with a* minute to spare: master shards ran up to 1.4x their prediction, and shard* 5/10 of run 39121354115 was cancelled at the timeout with no failing test.