Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Crash robot: runs generated op schedules through the real operation
* handlers on a managed brain and checks the reference model after every step,
* after every injected crash and recovery, and after the final drain.
*
* `runSchedule` executes one schedule inside the current process (the owner
* consumer runs here too). The process-separated crash mode in `worker.ts`
* reuses it: a worker installs a fault hook, the driver SIGKILLs it at the
* chosen point, and a fresh worker recovers and resumes the schedule.
*/
import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, utimesSync, writeFileSync, writeSync } from 'node:fs';
import { join } from 'node:path';
import type { BrainEngine } from '../../src/core/engine.ts';
import type { GBrainConfig } from '../../src/core/config.ts';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { sqlEngineKind } from '../../src/core/sql-engine-kind.ts';
import { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { disposePersistenceConsumer, startPersistenceConsumer } from '../../src/core/persistence/service.ts';
import { installFaultHook, type FaultDetail, type FaultPoint } from '../../src/core/persistence/fault-points.ts';
import { bigintToStringReplacer } from '../../src/core/utils.ts';
import { hardenedGitStream } from '../../src/core/hardened-git.ts';
import { __setSyncGitStreamForTests } from '../../src/core/persistence/sync-git-stream.ts';
import { assertSafeE2eDatabaseUrl } from '../../test/helpers/db-guard.ts';
import postgres from '#postgres';
import { prepareTopology } from './history-fixture.ts';
import { installLockOrderTrace, lockOrderReport } from './lock-order.ts';
import { executeOp, retryingRead, SessionDrops, type OpDescriptor, type OpObservation, type World } from './ops.ts';
import { KNOWN_DEFERRALS, ReferenceModel, SAFETY_CLASSES, type Violation } from './model.ts';
import type { Schedule } from './generator.ts';
import { pageBody } from './generator.ts';
import { descriptor } from './ops.ts';
import { randomUUID } from 'node:crypto';
export interface ScheduleResult { label: string; seed: number; steps: number; violations: Violation[]; safety: Violation[]; observations: OpObservation[] }
/**
* How long a drain may take before the robot calls it a wedge. On PGLite a
* restarted owner proves its predecessor exited (one process per datastore),
* so every claim must resume within 20 s. Postgres owners run concurrently and
* wait out a dead owner's claim lease (requests 30 s, effects 2 minutes), so
* the bound there is the effect lease plus margin.
*/
export const DRAIN_BOUND_MS = { pglite: 20_000, postgres: 140_000 } as const;
/** Wait until no request is pending and no effect is runnable; false when the bound passes (a wedge). */
export async function drain(engine: BrainEngine, world: World, timeoutMs = 60_000): Promise<boolean> {
startPersistenceConsumer(engine, world.config).wake();
const deadline = Date.now() + timeoutMs;
for (;;) {
const [row] = await retryingRead(world, () => engine.executeRaw<{ n: number }>(`SELECT
(SELECT count(*) FROM persistence_requests WHERE state IN ('queued','running','recovering'))
+ (SELECT count(*) FROM persistence_effects WHERE (state IN ('queued','running') AND next_attempt_at <= now() + interval '30 seconds') OR recovery IS NOT NULL) AS n`));
if (Number(row.n) === 0) return true;
if (Date.now() > deadline) return false;
await Bun.sleep(100);
}
}
/** Run `ops` from `start`, honoring concurrent groups; every observation is folded into `model`. */
export async function runSteps(world: World, model: ReferenceModel, schedule: Schedule, start = 0,
onStep?: (next: number, batch: OpDescriptor[]) => void): Promise<number> {
const groupOf = new Map<string, string[]>();
for (const group of schedule.groups) for (const id of group) groupOf.set(id, group);
let i = start;
while (i < schedule.ops.length) {
const d = schedule.ops[i];
const group = groupOf.get(d.id);
const batch = group ? schedule.ops.slice(i).filter(op => group.includes(op.id)) : [d];
onStep?.(i, batch);
const observed = await Promise.all(batch.map(op => executeOp(world, op)));
model.beginStep(batch.length > 1);
for (let k = 0; k < batch.length; k++) {
const op = batch[k];
await model.observe(op, observed[k], op.replayOf ? world.observations.get(op.replayOf) : undefined);
}
if (batch.length > 1) await model.settleGroup(batch, observed);
await model.checkGlobal(`after ${batch.map(op => op.id).join('+')}`);
i += batch.length;
}
return i;
}
/** Resubmit every request whose caller stopped waiting while it was pending; its final receipt is folded like a first observation. */
export async function settlePending(world: World, model: ReferenceModel): Promise<void> {
const pending = [...model.pending.values()];
if (!pending.length) return;
model.beginStep(true);
const observed: OpObservation[] = [];
for (const d of pending) {
const seen = await executeOp(world, d); observed.push(seen);
await model.observe({ ...d, replayOf: undefined }, seen);
if (seen.status === 'pending') model.violate({ class: 'wedge', op: d.id, detail: `request ${d.requestId} still pending after the drain` });
}
await model.settleLate(pending, observed);
}
/** Finish a schedule: drain, check drained invariants, prove a fresh write still admits, and drain that write's effects. */
export async function finish(world: World, model: ReferenceModel, label: string, drainMs: number = DRAIN_BOUND_MS[sqlEngineKind(world.engine.kind, 'crash robot')]): Promise<void> {
const drained = await drain(world.engine, world, drainMs);
if (!drained) model.violate({ class: 'wedge', detail: `${label}: requests or effects did not drain within ${drainMs / 1000} s` });
await settlePending(world, model);
await model.checkDrained(`${label}: drained`);
await model.checkGlobal(`${label}: drained`);
const source = world.remotes[0].sourceId;
const probe = descriptor(`probe-${randomUUID().slice(0, 8)}`, 'put_page', 'local', source,
{ slug: `notes/probe-${randomUUID().slice(0, 8)}`, content: pageBody('notes/probe', 'mk-probe0') });
const seen = await executeOp(world, probe);
if (seen.status !== 'committed') model.violate({ class: 'wedge', detail: `${label}: a fresh write after the schedule ended ${seen.status}/${seen.code}` });
await awaitClaimPass(world);
// The probe's own effects (its git commit among them) reach their seams only if the worker waits for them: a count
// role that happened to see the probe's effect:git:mid planned a fault no run was sure to reach (seed 5105).
if (!await drain(world.engine, world, drainMs)) model.violate({ class: 'wedge', detail: `${label}: the probe's effects did not drain within ${drainMs / 1000} s` });
}
/**
* #6957: wait for a consumer claim pass after the probe was prepared. The consumer makes one after every write it
* finishes, but a worker that returned at once stopped it first, unless timing let the pass in: a CI count role saw
* that pass as claim epoch 5 of caller_bound_replay, every replay of the run exited before making it, and its fault
* at consumer:claiming#5 went unreached. Waiting makes the last claim epoch part of every counted and run schedule.
* Only a worker whose fault hook counts claim passes (`world.claimPasses`) waits.
*/
async function awaitClaimPass(world: World): Promise<void> {
if (world.claimPasses === undefined) return;
const before = world.claimPasses;
const deadline = Date.now() + 30_000;
while (world.claimPasses === before && Date.now() < deadline) {
startPersistenceConsumer(world.engine, world.config).wake();
await Bun.sleep(50);
}
}
export async function runSchedule(world: World, schedule: Schedule): Promise<ScheduleResult> {
const model = new ReferenceModel(world);
world.descriptors = new Map(schedule.ops.map(d => [d.id, d]));
await runSteps(world, model, schedule);
await finish(world, model, schedule.label);
return { label: schedule.label, seed: schedule.seed, steps: schedule.ops.length, violations: model.violations,
safety: model.violations.filter(v => SAFETY_CLASSES.has(v.class)), observations: schedule.ops.map(d => world.observations.get(d.id)!) };
}
/* ------------------------------------------------------------------------ */
/* Process-separated crash mode: the worker side. */
/* ------------------------------------------------------------------------ */
export type ProcessFault = 'stale_index_lock' | 'hung_git' | 'pooler_disconnect' | 'facts_absorb_kill';
export interface RobotConfig {
kind: 'pglite' | 'postgres'; root: string; dataDir: string; databaseUrl?: string;
/** Direct (unpooled) URL of the run database, for administrative fault injection. */
directUrl?: string;
schedule: Schedule; worktrees: number; statePath: string;
/** SIGKILL at the nth time this seam is reached (1-based), optionally counting only one operation's requests. */
fault?: { point: FaultPoint; nth: number; operation?: string };
process?: ProcessFault;
}
interface RobotState {
remotes: World['remotes']; checkouts: string[]; roots?: World['roots']; connector?: World['connector']; next: number; inFlight: string[];
model: Record<string, unknown>; observations: [string, OpObservation][]; submitted: [string, Record<string, unknown>][];
}
export interface RobotOutcome extends Omit<ScheduleResult, 'observations'> {
crashed: boolean; fault?: RobotConfig['fault']; process?: ProcessFault; counts?: Record<string, number>; inFlight?: string[];
/** A run that finished without reaching its fault: how many slots of the fault's seam it made (#8078). */
reached?: number;
/** What each caller observed, kept only for a run with violations. */
trace?: string[];
/** Known findings outside the safety classes, filed in TODOS.md: reported, never failing the gate. */
deferred?: (Violation & { todo: string })[];
}
function writeState(config: RobotConfig, world: World, model: ReferenceModel, checkouts: string[], next: number, inFlight: string[]): void {
const state: RobotState = { remotes: world.remotes, checkouts, roots: world.roots, connector: world.connector, next, inFlight, model: model.toJSON(),
observations: [...world.observations], submitted: [...(world.submitted ?? [])] };
const staged = `${config.statePath}.tmp`;
writeFileSync(staged, JSON.stringify(state), { mode: 0o600 }); renameSync(staged, config.statePath);
}
async function openRobotEngine(config: RobotConfig, initialize: boolean): Promise<BrainEngine> {
const engine: BrainEngine = config.kind === 'pglite' ? new PGLiteEngine() : new PostgresEngine();
if (engine instanceof PostgresEngine) {
assertSafeE2eDatabaseUrl(config.databaseUrl!);
await engine.connect({ database_url: config.databaseUrl!, poolSize: 4 });
} else await engine.connect({ database_path: config.dataDir });
if (initialize) await engine.initSchema();
return engine;
}
/** Freeze the whole process at a seam until the driver's SIGKILL arrives; nothing after this point runs. */
function freeze(event: Record<string, unknown>): never {
const bytes = Buffer.from(`${JSON.stringify(event, bigintToStringReplacer)}\n`);
for (let offset = 0; offset < bytes.length;) offset += writeSync(1, bytes, offset, bytes.length - offset);
const blocked = new Int32Array(new SharedArrayBuffer(4));
for (;;) Atomics.wait(blocked, 0, 0);
}
/**
* Seams one request can reach more than once by design: a preparation runs
* again after a revision conflict or a group cut, as often as timing
* dictates (7 to 10 preparations for the same 8-op schedule), and a prepared
* request whose publication is requeued (`writer_lock_unavailable`) is
* prepared again (caller_bound_replay counted 4 or 5 `consumer:prepared`).
* Counting every pass made the count run and the run role disagree on nth,
* so a fault could go unreached. These seams count each write request row
* once (its first pass), keyed on `writeId`: a caller-bound replay's rows
* share one `requestId`.
*/
const PER_REQUEST_SEAMS: ReadonlySet<FaultPoint> = new Set<FaultPoint>(['consumer:preparing', 'consumer:prepared']);
/**
* #6896: `consumer:claiming` fires before every claim attempt, the consumer's idle ticks and the passes that find
* the root busy included, so how many passes a run makes follows wall-clock time, not the schedule (a count role
* saw 19, the run made fewer and its fault at #19 went unreached). It counts in claim epochs instead: the number of
* distinct write request rows that have reached `consumer:preparing`. The first pass at epoch e or later advances
* the count to e + 1, so slot n is the first claim attempt after n - 1 requests were prepared; requests of one root
* are claimed one after another, and `finish` waits for a claim pass after its probe, the last request (#6957), so
* every slot the count role saw comes again in the run.
*/
/**
* #7124: a sync pass makes as many rounds as the schedule's races dictate: a direct write still in flight when round 1
* reads its level defers that page to a round of its own, so the same schedule makes 2 or 3 rounds per pass (a count role
* saw 7 `sync:level_chunk`, a cold run made 6 and its fault at #7 went unreached). These seams count in round slots: in
* each pass, the first time a seam is reached in round 1, and the first time in a later round; a `sync:level_chunk`
* slot is one chunk ordinal of one Git read (`LEVEL_CHUNK_BYTES` makes the ordinals the same in every run). Every pass
* that made a round 1 with work makes a later round, its confirm round at least, so every slot comes again;
* `sync:batch_entries` counts in round 1 alone, since a later round has entries only when the race deferred one.
*/
const ROUND_SEAMS: ReadonlySet<FaultPoint> = new Set<FaultPoint>(['sync:round_start', 'sync:level_chunk', 'sync:level_result', 'sync:level_take',
'sync:level_read', 'sync:confirm_parallel', 'sync:level_narrow', 'sync:batch_entries']);
/** How far this pass advances the seam's count: 1, 0 for a repeat of the same request at a per-request seam or of a round slot, the epochs passed at the claim seam. */
export function seamCounter(): (point: FaultPoint, detail: FaultDetail) => number {
const seen = new Set<string>(), prepared = new Set<string>();
const chunks = new Map<string, number>();
let claimSlots = 0, pass = 0, round = 0;
return (point, detail) => {
if (point === 'sync:round_start') { round = Number(detail.round); if (round === 1) pass++; }
if (ROUND_SEAMS.has(point)) {
if (point === 'sync:batch_entries' && round > 1) return 0;
const read = `${pass}\u0000${round}\u0000${detail.read ?? ''}`;
const ordinal = point === 'sync:level_chunk' ? (chunks.get(read) ?? 0) + 1 : 0;
if (ordinal) chunks.set(read, ordinal);
const key = `${point}\u0000${pass}\u0000${round > 1 ? 'later' : 'first'}\u0000${detail.read ?? ''}\u0000${ordinal}`;
if (seen.has(key)) return 0;
seen.add(key);
return 1;
}
const request = detail.writeId ?? detail.requestId;
if (point === 'consumer:preparing' && request) prepared.add(request);
if (point === 'consumer:claiming') {
const advanced = Math.max(0, prepared.size + 1 - claimSlots);
claimSlots += advanced;
return advanced;
}
if (!PER_REQUEST_SEAMS.has(point) || !request) return 1;
const key = `${point}\u0000${request}`;
if (seen.has(key)) return 0;
seen.add(key);
return 1;
};
}
/**
* #7124: how many chunks a Git level read arrives in follows the pipe's timing, not the bytes read, so a count role and a
* run could see a read in different chunk counts. Robot workers re-cut every level read's stdout at fixed offsets of
* this many bytes (the last piece holds the rest): a read's chunk ordinals are then a function of what Git printed, and a
* robot tree of a few dozen paths reads in several chunks, so a crash between two chunks of one read is a slot like any.
*/
export const LEVEL_CHUNK_BYTES = 512;
export function fixedChunks(runner: typeof hardenedGitStream = hardenedGitStream): typeof hardenedGitStream {
return async (root, args, opts) => {
const deliver = opts.onChunk;
if (!deliver) return await runner(root, args, opts);
let rest = Buffer.alloc(0), chain = Promise.resolve();
const emit = (piece: Buffer) => { chain = chain.then(() => deliver(piece)); };
const result = await runner(root, args, { ...opts, onChunk: chunk => {
rest = Buffer.concat([rest, chunk]);
for (; rest.length >= LEVEL_CHUNK_BYTES; rest = rest.subarray(LEVEL_CHUNK_BYTES)) emit(Buffer.from(rest.subarray(0, LEVEL_CHUNK_BYTES)));
return chain;
} });
if (rest.length) emit(Buffer.from(rest));
await chain;
return result;
};
}
/**
* The `run` role's hook: freeze at the nth matching seam. A hook that throws
* would unwind into the code under test, which then keeps running as if no
* crash were requested (#6584), so the error is reported as a
* `fault_hook_error` event and the process freezes anyway; the driver counts
* such a run as a gate failure.
*/
export function robotFaultHook(fault: NonNullable<RobotConfig['fault']>): ((point: FaultPoint, detail: FaultDetail) => void) & { reached(): number } {
let seen = 0;
const counts = seamCounter();
return Object.assign((point: FaultPoint, detail: FaultDetail) => {
const advanced = counts(point, detail);
if (point !== fault.point || (fault.operation && detail.operation !== fault.operation) || !advanced) return;
seen += advanced;
if (seen < fault.nth) return;
try { freeze({ event: 'fault', point, nth: seen, detail }); }
catch (error) { freeze({ event: 'fault_hook_error', point, nth: seen, message: error instanceof Error ? error.message : String(error) }); }
}, { reached: () => seen });
}
function finalize(config: RobotConfig, model: ReferenceModel, extra: Partial<RobotOutcome>): RobotOutcome {
const locks = lockOrderReport();
for (const v of locks.violations) model.violate({ class: 'lock_order', detail: `${v.rule}: ${v.detail}` });
const trace = model.violations.length ? [...model.world.observations.values()].map(o =>
`${o.id} ${o.kind} ${o.actor}@${o.source} ${o.status}${o.code ? `/${o.code}` : ''}${o.values.revision ? ` rev=${o.values.revision}` : ''}`
+ `${o.receipt ? ` receipt=${o.receipt.state}` : ''}${o.status === 'refused' || o.kind === 'sync' || o.kind === 'connector_publish' ? ` ${JSON.stringify(o.raw).slice(0, 300)}` : ''}`) : undefined;
const deferredOf = (v: Violation) => KNOWN_DEFERRALS.find(k => k.class === v.class && k.match.test(v.detail));
const deferred = model.violations.filter(v => deferredOf(v)).map(v => ({ ...v, todo: deferredOf(v)!.todo }));
const violations = model.violations.filter(v => !deferredOf(v));
return { label: config.schedule.label, seed: config.schedule.seed, steps: config.schedule.ops.length, crashed: false,
violations, safety: violations.filter(v => SAFETY_CLASSES.has(v.class)), ...extra, ...(trace ? { trace } : {}),
...(deferred.length ? { deferred } : {}) };
}
/**
* A git shim whose `git commit` (or `commit-tree`, the Git effect's commit) hangs while the marker file exists, holding the
* checkout's index.lock the way a real stuck git does: SIGTERM lets it remove
* its lock and exit, SIGKILL leaves the lock behind.
*/
function installHungGit(root: string): { marker: string; release(): void } {
const dir = join(root, 'hung-git-bin'); mkdirSync(dir, { recursive: true });
const real = Bun.which('git')!; const marker = join(root, 'hung-git.marker');
writeFileSync(join(dir, 'git'), `#!/bin/sh
repo=.; prev=; commit=
for a in "$@"; do [ "$prev" = -C ] && repo="$a"; case "$a" in commit|commit-tree) commit=1;; esac; prev="$a"; done
if [ -n "$commit" ] && [ -e '${marker}' ]; then
lock="$repo/.git/index.lock"; : > "$lock"; echo $$ >> '${marker}.pids'
trap 'kill $! 2>/dev/null; rm -f "$lock"; exit 143' TERM
sleep 600 >/dev/null 2>&1 & wait
exit 1
fi
exec '${real}' "$@"
`);
chmodSync(join(dir, 'git'), 0o755); writeFileSync(marker, '');
process.env.PATH = `${dir}:${process.env.PATH}`;
return { marker, release: () => rmSync(marker, { force: true }) };
}
const ABSORB_SLUG = 'meetings/robot-review';
const ABSORB_FACTS = ['Robot review ships the crash gate weekly.', 'Robot review keeps withdrawn facts out of recall.'];
/** Deterministic providers for facts extraction: a chat stub that names two facts about the review, a constant embedding. */
async function stubFactsProviders(): Promise<void> {
const gateway = await import('../../src/core/ai/gateway.ts');
gateway.configureGateway({ embedding_model: 'openai:text-embedding-3-large', embedding_dimensions: 1536, env: { OPENAI_API_KEY: 'synthetic-local-fixture' } } as never);
gateway.__setChatTransportForTests((async () => ({ text: JSON.stringify({ facts: ABSORB_FACTS.map(fact => ({ fact, kind: 'fact', entity: ABSORB_SLUG, confidence: 0.9, notability: 'high' })) }),
blocks: [], stopReason: 'end', usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'test:stub', providerId: 'test' })) as never);
gateway.__setEmbedTransportForTests((async ({ values }: { values: string[] }) => ({ embeddings: values.map(() => [1, ...Array(1535).fill(0)]) })) as never);
}
/** Run the queued facts-absorb job through its registered handler, as a worker does after claiming it. */
async function runAbsorbJob(engine: BrainEngine, attempts: number): Promise<unknown> {
const [job] = await retryingRead({}, () => engine.executeRaw<{ id: number; data: Record<string, unknown> }>("SELECT id,data FROM minion_jobs WHERE name='facts-absorb' ORDER BY id LIMIT 1"));
if (!job) throw new Error('facts_absorb_kill: no facts-absorb job was queued');
const { MinionWorker } = await import('../../src/core/minions/worker.ts');
const { registerBuiltinHandlers } = await import('../../src/commands/jobs.ts');
const worker = new MinionWorker(engine, { queue: 'crash-robot' });
await registerBuiltinHandlers(worker, engine, { quiet: true });
const signal = new AbortController().signal;
return worker.getHandler('facts-absorb')!({ id: Number(job.id), name: 'facts-absorb', data: job.data, attempts_made: attempts, signal, deadlineAtMs: null,
shutdownSignal: signal, updateProgress: async () => {}, updateTokens: async () => {}, log: async () => {}, isActive: async () => true, readInbox: async () => [] } as never);
}
/** Every absorbed fact is active exactly once. */
async function checkAbsorbedOnce(model: ReferenceModel, stage: string): Promise<void> {
const rows = await model.q<{ fact: string; n: number }>(`SELECT fact,count(*)::int AS n FROM facts WHERE fact = ANY($1::text[])
AND expired_at IS NULL GROUP BY fact`, [ABSORB_FACTS]);
for (const row of rows) if (row.n > 1) model.violate({ class: 'duplicate_apply', detail: `${stage}: "${row.fact}" is active ${row.n} times` });
if (rows.length !== ABSORB_FACTS.length) model.violate({ class: 'lost_write', detail: `${stage}: ${rows.length}/${ABSORB_FACTS.length} absorbed facts are active` });
}
/**
* Process faults SIGKILL cannot model. Each must end in a terminal state or a
* typed, agent-facing error, never a wedge, and must clear once the fault does.
*/
async function processFault(config: RobotConfig, world: World, model: ReferenceModel, checkouts: string[]): Promise<void> {
const engine = world.engine;
const typedGitErrors = new Set(['git_index_locked', 'git_index_stale', 'git_failed', 'git_timeout', 'targets_parked', 'storage_error']);
const gitEffects = () => engine.executeRaw<{ id: string; state: string; error_code: string | null }>(
"SELECT id::text,state,error_code FROM persistence_effects WHERE kind='git' AND state<>'committed'");
if (config.process === 'stale_index_lock') {
const lock = join(checkouts[0], '.git', 'index.lock'); writeFileSync(lock, '');
const old = new Date(Date.now() - 3_600_000); utimesSync(lock, old, old);
await runSteps(world, model, config.schedule);
const deadline = Date.now() + 30_000;
let stuck = await gitEffects();
while (Date.now() < deadline && stuck.some(e => e.state === 'running' || !typedGitErrors.has(e.error_code ?? ''))) { await Bun.sleep(250); stuck = await gitEffects(); }
for (const e of stuck) if (!e.error_code || !typedGitErrors.has(e.error_code)) {
model.violate({ class: 'wedge', detail: `stale index.lock: git effect ${e.id} is ${e.state} without a typed error (${e.error_code})` });
}
rmSync(lock, { force: true });
await engine.executeRaw("UPDATE persistence_effects SET next_attempt_at=now() WHERE kind='git' AND state='queued'");
} else if (config.process === 'hung_git') {
const shim = installHungGit(config.root);
await runSteps(world, model, config.schedule);
const deadline = Date.now() + 15_000;
while (Date.now() < deadline && !existsSync(`${shim.marker}.pids`)) await Bun.sleep(100);
if (existsSync(`${shim.marker}.pids`)) {
const started = Date.now();
const stopped = await Promise.race([disposePersistenceConsumer(engine).then(() => true), Bun.sleep(45_000).then(() => false)]);
if (!stopped) model.violate({ class: 'wedge', detail: 'hung git child: consumer shutdown did not finish within 45 s' });
else if (Date.now() - started > 30_000) model.violate({ class: 'wedge', detail: `hung git child: consumer shutdown took ${Date.now() - started} ms` });
} else model.violate({ class: 'wedge', detail: 'hung_git: the hung commit never started' });
shim.release();
await engine.executeRaw("UPDATE persistence_effects SET next_attempt_at=now() WHERE kind='git' AND state='queued'");
} else if (config.process === 'facts_absorb_kill') {
// A substantive meeting page queues facts-backstop work; the worker runs the facts-absorb job and is
// SIGKILLed right after its extraction commits, before the job completes (config.fault).
await stubFactsProviders();
const source = world.remotes[0].sourceId;
const body = 'The robot review covered the crash gate, its seams and the withdrawal checks in detail. '.repeat(4);
const put = await executeOp(world, { v: 1, id: 'absorb-page', kind: 'put_page', actor: 'local', source, requestId: crypto.randomUUID(),
args: { slug: ABSORB_SLUG, content: `---\ntype: meeting\ntitle: Robot review\n---\n\n${body}\n` } });
if (put.status !== 'committed') throw new Error(`facts_absorb_kill: the meeting page did not commit (${put.code})`);
const deadline = Date.now() + 30_000;
while (!(await retryingRead(world, () => engine.executeRaw("SELECT 1 FROM minion_jobs WHERE name='facts-absorb'"))).length) {
if (Date.now() > deadline) throw new Error('facts_absorb_kill: the facts-backstop effect never queued a facts-absorb job');
await Bun.sleep(100);
}
await runAbsorbJob(engine, 0);
await checkAbsorbedOnce(model, 'facts_absorb_kill (no crash reached)');
} else if (config.process === 'pooler_disconnect') {
// Every 400 ms the server drops every other session of this database, as a pooler or failover does.
// Each round is recorded, so harness reads retry only the closes this fault caused.
const admin = postgres(config.directUrl!, { max: 1, onnotice() {} });
const drops = world.sessionDrops = new SessionDrops();
let stop = false; let dropped = 0;
const dropping = (async () => {
while (!stop) {
await Bun.sleep(400);
dropped += await drops.inject(() => admin.unsafe(`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
WHERE datname=current_database() AND pid<>pg_backend_pid()`).then(rows => rows.length));
}
})();
try { await runSteps(world, model, config.schedule); }
finally { stop = true; await dropping; await admin.end({ timeout: 1 }); }
if (!dropped) model.violate({ class: 'wedge', detail: 'pooler_disconnect: no session was dropped; the fault did not run' });
}
await finish(world, model, `${config.schedule.label}+${config.process}`);
}
/**
* Worker entry for one crash-robot role. `count` runs the schedule with a
* counting hook; `run` starts a fresh brain and freezes at `config.fault`;
* `recover` reopens the brain after the driver's SIGKILL, checks the
* pre-recovery state, drains, resubmits the in-flight requests and finishes.
*/
export async function robotWorker(role: 'count' | 'run' | 'recover', config: RobotConfig): Promise<RobotOutcome> {
const fresh = role !== 'recover';
const engine = await openRobotEngine(config, fresh);
try {
let world: World; let model: ReferenceModel; let checkouts: string[]; let state: RobotState | undefined;
if (fresh) {
const topology = await prepareTopology(engine, { sources: 2, worktrees: config.worktrees, root: join(config.root, 'checkouts'), prefix: 'robot', connector: true });
world = topology.world; checkouts = topology.checkouts; model = new ReferenceModel(world);
} else {
state = JSON.parse(readFileSync(config.statePath, 'utf8')) as RobotState;
world = { engine, config: { engine: engine.kind, embedding_disabled: true } as GBrainConfig, remotes: state.remotes, auth: new Map(),
observations: new Map(state.observations), submitted: new Map(state.submitted), roots: state.roots, connector: state.connector };
checkouts = state.checkouts; model = ReferenceModel.fromJSON(world, state.model);
}
world.descriptors = new Map(config.schedule.ops.map(d => [d.id, d]));
installLockOrderTrace();
__setSyncGitStreamForTests(fixedChunks());
const counts: Record<string, number> = {};
const counted = seamCounter();
const faultHook = role === 'run' && config.fault ? robotFaultHook(config.fault) : undefined;
const hook = role === 'count' ? (point: FaultPoint, detail: FaultDetail) => { const advanced = counted(point, detail); if (advanced) counts[point] = (counts[point] ?? 0) + advanced; }
: faultHook;
if (hook) {
world.claimPasses = 0;
installFaultHook((point, detail) => { if (point === 'consumer:claiming') world.claimPasses!++; hook(point, detail); });
}
if (role === 'run') {
let next = 0; let inFlight: string[] = [];
world.onSubmit = () => writeState(config, world, model, checkouts, next, inFlight);
writeState(config, world, model, checkouts, 0, []);
if (config.process) { await processFault(config, world, model, checkouts); return finalize(config, model, { process: config.process }); }
await runSteps(world, model, config.schedule, 0, (i, batch) => { next = i; inFlight = batch.map(d => d.id); writeState(config, world, model, checkouts, next, inFlight); });
await finish(world, model, config.schedule.label);
return finalize(config, model, { fault: config.fault, ...(faultHook ? { reached: faultHook.reached() } : {}) });
}
if (role === 'count') {
await runSteps(world, model, config.schedule);
await finish(world, model, config.schedule.label);
return finalize(config, model, { counts });
}
// recover
const inFlight = state!.inFlight.map(id => world.descriptors!.get(id)!);
model.beginStep(inFlight.length > 1);
model.allowInFlight(inFlight); model.relaxInFlight(inFlight);
await model.checkGlobal('after SIGKILL, before recovery');
const bound = DRAIN_BOUND_MS[sqlEngineKind(engine.kind, 'crash robot')];
const drained = await drain(engine, world, bound);
if (!drained) model.violate({ class: 'wedge', detail: `recovery: requests or effects did not drain within ${bound / 1000} s after restart` });
await model.checkDrained('recovered');
await settlePending(world, model);
model.allowInFlight(inFlight);
await model.checkGlobal('recovered');
// The interrupted callers resubmit with their original request ids.
if (inFlight.length) {
const observed = await Promise.all(inFlight.map(d => executeOp(world, d)));
model.beginStep(true);
for (let k = 0; k < inFlight.length; k++) await model.observe(inFlight[k], observed[k], inFlight[k].replayOf ? world.observations.get(inFlight[k].replayOf!) : undefined);
// A refused resubmission says nothing about the interrupted attempt, which may have committed.
for (let k = 0; k < inFlight.length; k++) if (observed[k].status === 'refused') model.uncertain(inFlight[k]);
await model.settleLate(inFlight, observed);
await model.checkGlobal('after resubmission');
}
if (config.process === 'facts_absorb_kill') {
// The job's lease expired with its owner; a worker runs it again.
await stubFactsProviders();
await runAbsorbJob(engine, 1);
await checkAbsorbedOnce(model, 'facts-absorb re-run after SIGKILL');
await finish(world, model, `${config.schedule.label}+facts_absorb_kill`);
return finalize(config, model, { crashed: true, fault: config.fault, process: config.process });
}
await runSteps(world, model, config.schedule, state!.next + inFlight.length);
await finish(world, model, config.schedule.label);
return finalize(config, model, { crashed: true, fault: config.fault, inFlight: state!.inFlight });
} finally {
installFaultHook(undefined);
__setSyncGitStreamForTests();
await disposePersistenceConsumer(engine).catch(() => {});
await engine.disconnect();
}
}