Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.7 KB
Raw Blame
/**
 * Driver for the crash robot phase of the persistence gate. For each
 * schedule (the cross-boundary sequences first, then the fixed list of seeded
 * random sequences, ROBOT_RANDOM_SEEDS plus any exploration seeds) it runs a counting pass that
 * finds every seam the schedule reaches, then one process-separated run per
 * chosen crash point: a worker freezes at the seam, the driver SIGKILLs it,
 * and a fresh worker recovers, resubmits the interrupted requests and
 * finishes the schedule. Every run is checked by the reference model.
 * Process faults (stale index.lock, hung git child) run once per engine.
 */
import { createHash, randomUUID } from 'node:crypto';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import type postgres from '#postgres';
import { random } from './harness.ts';
import { crossBoundarySequences, randomSchedule, type GeneratorTopology, type Schedule } from './generator.ts';
import type { ProcessFault, RobotConfig, RobotOutcome } from './crash-robot.ts';
import type { FaultPoint } from '../../src/core/persistence/fault-points.ts';
import type { OpDescriptor } from './ops.ts';
import type { spawnWorker as SpawnWorker } from './validate.ts';

export const ROBOT_TOPOLOGY: GeneratorTopology = { sources: ['robot-0', 'robot-1'],
  remotes: [{ name: 'agent-oauth', sourceId: 'robot-0' }, { name: 'agent-token', sourceId: 'robot-1' }], connector: 'robot-gh' };
export const PROCESS_FAULTS: ProcessFault[] = ['stale_index_lock', 'hung_git', 'pooler_disconnect'];

/** One executed run, enough to replay it exactly: `--replay` re-injects this entry. */
/**
 * Seams and process faults whose recovery on Postgres can wait out a dead
 * owner's claim lease (requests 30 s, effects 2 minutes): a publication seam
 * does when the dead owner also held an effect claim, and the pooler
 * disconnect fault holds its run for about two minutes. A Postgres run with a
 * run of the `pr` scope skips them and lists them in the manifest
 * (`lease_bound_seams_skipped`, `lease_bound_faults_skipped`); full runs
 * (master, nightly, manual) crash every seam and run every fault.
 */
export type RobotScope = 'pr' | 'full';
/**
 * The random sequences every run executes, whatever its event, so two runs of
 * one tree cover the same schedules and a passed run can be reused for an
 * identical tree. 5108, 5109, 5113, 5115 and 5116 are the shrunk
 * fence_absent fixtures of #7435.
 */
export const ROBOT_RANDOM_SEEDS: readonly number[] = [5105, 5106, 5107, 5108, 5109, 5113, 5115, 5116];
/**
 * Runs in flight at once. Every run owns its database, scratch root and home, and the plan is fixed before any
 * crash runs, so the width changes only the wall clock (#8036). Two is measured on ubicloud-standard-4
 * (docs/ci-event-parity.md#crash-robot-ceilings).
 */
export const ROBOT_CONCURRENCY = 2;
/** Seeds from 5109 to 5116 not in ROBOT_RANDOM_SEEDS: the nightly runs one of them each night. */
export const ROBOT_ROTATION_SEEDS: readonly number[] = [5110, 5111, 5112, 5114];
/** The nightly's exploration seeds for a UTC day number: one rotation seed (each within four nights) and one never run before. */
export function explorationSeeds(day: number): number[] {
  return [ROBOT_ROTATION_SEEDS[day % ROBOT_ROTATION_SEEDS.length]!, 5117 + day];
}
const LEASE_BOUND_PUBLICATION_SEAMS: readonly FaultPoint[] = ['publication:prepared', 'publication:before_publication', 'publication:after_publication'];
export function leaseBoundSeam(point: FaultPoint): boolean {
  return point.startsWith('effect:') || point === 'consumer:prepared' || LEASE_BOUND_PUBLICATION_SEAMS.includes(point);
}
export const LEASE_BOUND_FAULTS: readonly ProcessFault[] = ['pooler_disconnect'];

export interface RobotRun {
  schedule: string; seed: number; length: number; fault?: { point: FaultPoint; nth: number }; process?: ProcessFault;
  ops?: string[]; crashed: boolean; violations: RobotOutcome['violations']; duration_ms: number; trace?: string[];
  deferred?: RobotOutcome['deferred'];
  /** The worker finished without reaching its requested fault (#6584): nothing was crashed or recovered. */
  fault_not_reached?: boolean;
  /** Slots of the fault's seam the schedule's count run made, when the fault was planned from one. */
  counted?: number;
  /** Slots of the fault's seam made by a run that missed its fault. */
  reached?: number;
  /** The fault this run was planned with, and how many slots the run that missed it made (see `retargetable`). */
  retargeted?: { nth: number; reached: number };
  /** The worker's fault hook threw at the requested seam (`fault_hook_error`); the run proves nothing. */
  hook_error?: string;
}

/**
 * Requested faults a scenario (a schedule label, or a process fault) may
 * legitimately finish without reaching, each with its reason. Any other
 * unreached fault fails the run and the full gate.
 */
export const UNREACHED_FAULT_ALLOWLIST: readonly { scenario: string; point: FaultPoint; reason: string }[] = [];
export function unreachedAllowed(run: Pick<RobotRun, 'schedule' | 'process' | 'fault'>): boolean {
  return UNREACHED_FAULT_ALLOWLIST.some(entry => entry.scenario === (run.process ?? run.schedule) && entry.point === run.fault?.point);
}
/**
 * #8078: a schedule whose concurrent group settles one of two legal ways makes one slot more or fewer at some seams
 * (sync_and_connector_race_direct_write: the connector republish racing a direct write publishes 5 or 6 times, and
 * its git effects, preparations and commits follow), so a fault the count run planned at its last slot can be missed
 * by a run that took the other outcome. Such a run, short by exactly one slot of the count run, runs once more at the
 * last slot it did make. A run that found violations, made no slot, fell more than one short, or missed again fails as before.
 */
export function retargetable(run: RobotRun): boolean {
  const reached = run.reached ?? 0;
  return run.fault_not_reached === true && !run.retargeted && run.counted !== undefined && run.fault !== undefined && run.violations.length === 0
    && reached > 0 && reached >= run.counted - 1 && reached < run.fault.nth;
}
/** A run fails the gate when it found violations, its hook threw, or it never reached a fault it was not allowed to miss. */
export function runFailed(run: RobotRun): boolean {
  return run.violations.length > 0 || run.hook_error !== undefined || (run.fault_not_reached === true && !unreachedAllowed(run));
}
export interface RobotOptions {
  /** `seconds` is the ceiling: a run past it is reported `over_budget`. */
  engine: 'pglite' | 'postgres'; seed: number; seconds: number; scratch: string; home: string;
  /** `full` (default) crashes every seam; `pr` skips the Postgres lease-bound ones. */
  scope?: RobotScope;
  /** Random seeds run after ROBOT_RANDOM_SEEDS (the nightly's explorationSeeds). */
  extraSeeds?: readonly number[];
  admin?: ReturnType<typeof postgres>; databaseUrl?: string; databases: string[];
  /** A transaction-mode PgBouncer URL: Postgres workers then connect through it (prepared statements off). */
  pooledUrl?: string;
  spawn: typeof SpawnWorker; track: (child: ReturnType<typeof SpawnWorker>) => void;
  /** Re-run exactly these entries instead of generating. */
  replay?: RobotRun[];
  /** Keep only these op ids of a schedule (shrinking). */
  keep?: Map<string, Set<string>>;
  worktrees?: number;
  /** Runs in flight at once (default ROBOT_CONCURRENCY); a replay runs one at a time. */
  concurrency?: number;
  log?: (line: string) => void;
}

export function scheduleFor(label: string, seed: number, length: number): Schedule {
  if (label.startsWith('random-')) return randomSchedule(ROBOT_TOPOLOGY, seed, length);
  const found = crossBoundarySequences(ROBOT_TOPOLOGY, seed).find(s => s.label === label);
  if (!found) throw new Error(`Unknown crash-robot schedule ${label}`);
  return found;
}
/** Drop ops not in `keep`, and any op whose dependencies were dropped; groups shrink with them. */
export function restrict(schedule: Schedule, keep?: Set<string>): Schedule {
  if (!keep) return schedule;
  const kept = new Set<string>();
  const ops: OpDescriptor[] = [];
  for (const d of schedule.ops) {
    if (!keep.has(d.id)) continue;
    if ((d.deps ?? []).some(dep => !kept.has(dep) && schedule.ops.some(o => o.id === dep))) continue;
    kept.add(d.id); ops.push(d);
  }
  return { ...schedule, ops, groups: schedule.groups.map(g => g.filter(id => kept.has(id))).filter(g => g.length > 1) };
}

export async function runRobotPhase(o: RobotOptions) {
  const started = performance.now();
  const runs: RobotRun[] = [];
  const log = o.log ?? (line => process.stderr.write(`[crash-robot] ${o.engine}: ${line}\n`));
  async function fresh(name: string): Promise<{ root: string; dataDir: string; databaseUrl?: string; directUrl?: string; database?: string }> {
    const root = join(o.scratch, `robot-${name}-${randomUUID().slice(0, 8)}`); mkdirSync(root, { recursive: true });
    if (!o.admin) return { root, dataDir: join(root, 'data') };
    const database = `gbrain_persistence_test_${randomUUID().replaceAll('-', '')}`;
    await o.admin.unsafe(`CREATE DATABASE ${database}`); o.databases.push(database);
    const direct = new URL(o.databaseUrl!); direct.pathname = `/${database}`;
    const pooled = o.pooledUrl ? new URL(o.pooledUrl) : null; if (pooled) pooled.pathname = `/${database}`;
    return { root, dataDir: join(root, 'data'), databaseUrl: (pooled ?? direct).toString(), directUrl: direct.toString(), database };
  }
  // A transaction-mode pooler keeps every server connection a run opened in
  // that run's per-database pool until its own idle timeout (PgBouncer: 600 s,
  // the length of a full robot budget), so a run's database is released the
  // moment the run passes: DROP ... WITH (FORCE) ends those backends. A
  // failing run keeps its database for the retained-fixture metadata.
  async function release(database: string | undefined) {
    if (!database || !o.admin) return;
    await o.admin.unsafe(`DROP DATABASE IF EXISTS ${database} WITH (FORCE)`);
    const index = o.databases.indexOf(database); if (index >= 0) o.databases.splice(index, 1);
  }
  const environment: Record<string, string> = o.pooledUrl ? { GBRAIN_PREPARE: 'false' } : {};
  async function execute(schedule: Schedule, role: 'count' | 'run', extra: Partial<RobotConfig> = {}, planned: Pick<RobotRun, 'counted' | 'retargeted'> = {}): Promise<RobotRun & { counts?: Record<string, number> }> {
    const at = performance.now();
    const { database, ...place } = await fresh(`${schedule.label}-${role}`);
    const config: RobotConfig = { kind: o.engine, ...place, schedule, worktrees: o.worktrees ?? 2, statePath: join(place.root, 'state.json'), ...extra };
    const path = join(place.root, 'robot.json'); writeFileSync(path, JSON.stringify(config), { mode: 0o600 });
    const home = join(o.home, basename(place.root)); mkdirSync(home, { recursive: true });
    const base = { schedule: schedule.label, seed: schedule.seed, length: schedule.ops.length, ...(extra.fault ? { fault: extra.fault } : {}),
      ...(extra.process ? { process: extra.process } : {}), ...(o.keep?.has(schedule.label) ? { ops: schedule.ops.map(d => d.id) } : {}), ...planned };
    const child = o.spawn(path, home, 'robot', [role], environment); o.track(child);
    const outcome = async (): Promise<RobotRun & { counts?: Record<string, number> }> => {
      if (role === 'run' && extra.fault) {
        const reached = await Promise.race([child.event('fault', 600_000).then(e => ({ fault: e })),
          child.event('fault_hook_error', 600_000).then(e => ({ hookError: e })), child.event('done', 600_000).then(e => ({ done: e }))]);
        if ('hookError' in reached) {
          await child.kill();
          return { ...base, crashed: false, hook_error: String(reached.hookError.message), violations: [], duration_ms: performance.now() - at };
        }
        if ('done' in reached) {
          const result = reached.done.result as RobotOutcome;
          return { ...base, crashed: false, fault_not_reached: true, ...(result.reached !== undefined ? { reached: result.reached } : {}),
            violations: result.violations, deferred: result.deferred, trace: result.trace, duration_ms: performance.now() - at };
        }
        await child.kill();
        const recovered = o.spawn(path, home, 'robot', ['recover'], environment); o.track(recovered);
        const result = (await recovered.done()).result as RobotOutcome;
        return { ...base, crashed: true, violations: result.violations, deferred: result.deferred, trace: result.trace, duration_ms: performance.now() - at };
      }
      const result = (await child.done()).result as RobotOutcome;
      return { ...base, crashed: false, violations: result.violations, deferred: result.deferred, trace: result.trace, counts: result.counts, duration_ms: performance.now() - at };
    };
    const run = await outcome();
    if (retargetable(run)) {
      await release(database);
      log(`${run.schedule} ${run.fault!.point}#${run.fault!.nth}: the run made ${run.reached} of the count run's ${run.counted} slots; crashing again at #${run.reached}`);
      return execute(schedule, role, { ...extra, fault: { ...extra.fault!, nth: run.reached! } }, { counted: run.counted, retargeted: { nth: run.fault!.nth, reached: run.reached! } });
    }
    if (!runFailed(run)) await release(database);
    return run;
  }
  const say = (run: RobotRun) => {
    const what = run.fault ? `${run.fault.point}#${run.fault.nth}` : run.process ?? 'no crash';
    const verdict = run.violations.length ? `${run.violations.length} VIOLATION(S) ${run.violations.map(v => v.class).join(',')}`
      : run.hook_error !== undefined ? `FAULT HOOK ERROR ${run.hook_error}`
        : run.fault_not_reached ? (unreachedAllowed(run) ? 'fault not reached (allowed)' : 'FAULT NOT REACHED') : 'ok';
    log(`${run.schedule} ${what}: ${verdict} (${Math.round(run.duration_ms)} ms)`);
  };
  const record = (run: RobotRun) => { runs.push(run); say(run); };
  const go = (schedule: Schedule, role: 'count' | 'run', extra: Partial<RobotConfig> = {}, planned: Pick<RobotRun, 'counted'> = {}) =>
    execute(schedule, role, extra, planned).then(run => { say(run); return run; });
  const width = o.concurrency ?? ROBOT_CONCURRENCY;
  /** `each` over `items`, `width` at a time, each result at its item's index. The first failure stops every lane from taking another item. */
  async function pooled<T, R>(items: readonly T[], each: (item: T) => Promise<R>): Promise<R[]> {
    const results: R[] = [];
    let next = 0;
    let stopped = false;
    const lane = async () => {
      while (!stopped && next < items.length) {
        const index = next++;
        try { results[index] = await each(items[index]!); } catch (error) { stopped = true; throw error; }
      }
    };
    await Promise.all(Array.from({ length: Math.min(width, items.length) }, lane));
    return results;
  }

  if (o.replay) {
    for (const entry of o.replay) {
      const schedule = restrict(scheduleFor(entry.schedule, entry.seed, entry.length), entry.ops ? new Set(entry.ops) : o.keep?.get(entry.schedule));
      record(await execute(schedule, entry.fault || entry.process ? 'run' : 'count', { ...(entry.fault ? { fault: entry.fault } : {}), ...(entry.process ? { process: entry.process } : {}) },
        entry.counted !== undefined ? { counted: entry.counted } : {}));
    }
    return summarize(runs, started, o);
  }

  const rand = random(o.seed);
  const scope = o.scope ?? 'full';
  const fixed = crossBoundarySequences(ROBOT_TOPOLOGY, o.seed);
  const seeded = [...ROBOT_RANDOM_SEEDS, ...(o.extraSeeds ?? [])].map(seed => randomSchedule(ROBOT_TOPOLOGY, seed >>> 0, 24));
  // Every schedule is counted first and every fault is then drawn in one fixed order, so how many runs are in
  // flight never changes which runs execute, and the manifest lists them in that order.
  const counted = await pooled([...fixed, ...seeded], schedule => go(schedule, 'count'));
  const fixedCounts = counted.slice(0, fixed.length);
  const seededCounts = counted.slice(fixed.length);
  // Every cross-boundary sequence runs uncrashed, then each seam it reaches is crashed once,
  // seams first covered across sequences (a seam no run has crashed yet goes first).
  const plan: { schedule: Schedule; point: FaultPoint; count: number }[] = [];
  const skipLease = o.engine === 'postgres' && scope === 'pr';
  // A full PGLite run crashes every seam of every sequence; Postgres and pull requests crash each seam once.
  const everySeam = scope === 'full' && o.engine === 'pglite';
  const skipped = new Set<string>();
  fixed.forEach((schedule, index) => {
    for (const [point, count] of Object.entries(fixedCounts[index]!.counts ?? {}) as [FaultPoint, number][]) {
      if (skipLease && leaseBoundSeam(point)) { skipped.add(point); continue; }
      plan.push({ schedule, point, count });
    }
  });
  // The seed rotates which sequence covers each seam first.
  const turn = (x: (typeof plan)[number]) => (fixed.indexOf(x.schedule) + o.seed) % fixed.length;
  plan.sort((x, y) => x.point.localeCompare(y.point) || turn(x) - turn(y));
  const firsts = plan.filter((x, i) => plan.findIndex(y => y.point === x.point) === i);
  const ordered = [...firsts, ...plan.filter(x => !firsts.includes(x))];
  type Job = { schedule: Schedule; extra: Partial<RobotConfig>; counted?: number };
  const crashes: Job[] = (everySeam ? ordered : firsts)
    .map(item => ({ schedule: item.schedule, extra: { fault: { point: item.point, nth: 1 + Math.floor(rand() * item.count) } }, counted: item.count }));
  const skippedFaults = new Set<ProcessFault>();
  const processes: Job[] = [];
  for (const fault of PROCESS_FAULTS) {
    if (fault === 'pooler_disconnect' && !o.admin) continue;
    if (skipLease && LEASE_BOUND_FAULTS.includes(fault)) { skippedFaults.add(fault); continue; }
    processes.push({ schedule: fixed[1]!, extra: { process: fault } });
  }
  // serve killed right after the facts-absorb job's extraction commits; the job then runs again.
  processes.push({ schedule: { ...fixed[1]!, ops: [], groups: [] }, extra: { process: 'facts_absorb_kill',
    fault: { point: 'publication:after_commit', nth: 1, operation: 'extract_facts' } } });
  // The fixed random sequences, each counted and crashed once. On Postgres the crash avoids the lease-bound
  // seams: the cross-boundary sequences already crash each of them once.
  const seededCrashes = seeded.map((schedule, index): Job | undefined => {
    const points = (Object.entries(seededCounts[index]!.counts ?? {}) as [FaultPoint, number][]).filter(([point]) => !(o.engine === 'postgres' && leaseBoundSeam(point)));
    if (!points.length) return undefined;
    const [point, count] = points[Math.floor(rand() * points.length)]!;
    return { schedule, extra: { fault: { point, nth: 1 + Math.floor(rand() * count) } }, counted: count };
  });
  // The process faults start first: a hung git child and a dead owner's claim lease mostly wait, so they
  // overlap the crash runs instead of adding their wait to the wall clock.
  const jobs = [...processes, ...crashes, ...seededCrashes.filter((job): job is Job => job !== undefined)];
  const finished = await pooled(jobs, job => go(job.schedule, 'run', job.extra, job.counted !== undefined ? { counted: job.counted } : {}));
  const result = (job: Job) => finished[jobs.indexOf(job)]!;
  runs.push(...fixedCounts, ...crashes.map(result), ...processes.map(result));
  seeded.forEach((_, index) => { runs.push(seededCounts[index]!); const job = seededCrashes[index]; if (job) runs.push(result(job)); });
  const summary = summarize(runs, started, o);
  return { ...summary, scope, over_budget: o.seconds > 0 && summary.duration_ms > o.seconds * 1000,
    lease_bound_seams_skipped: [...skipped].sort(), lease_bound_faults_skipped: [...skippedFaults].sort() };
}

function summarize(runs: RobotRun[], started: number, o: RobotOptions) {
  const crashRuns = runs.filter(r => r.fault);
  const points = new Set(crashRuns.map(r => r.fault!.point));
  const violations = runs.flatMap(r => r.violations.map(v => ({ ...v, schedule: r.schedule, seed: r.seed, fault: r.fault, process: r.process })));
  return {
    seed: o.seed, budget_seconds: o.seconds, duration_ms: performance.now() - started,
    schedules: new Set(runs.map(r => `${r.schedule}:${r.seed}`)).size,
    crash_runs: crashRuns.length, crashed_runs: crashRuns.filter(r => r.crashed).length,
    fault_hook_errors: runs.filter(r => r.hook_error !== undefined),
    faults_not_reached: runs.filter(r => r.fault_not_reached && !unreachedAllowed(r)),
    faults_not_reached_allowed: runs.filter(r => r.fault_not_reached && unreachedAllowed(r)),
    faults_retargeted: runs.filter(r => r.retargeted),
    sequences_x_crash_points: crashRuns.length, distinct_crash_points: [...points].sort(),
    process_faults: runs.filter(r => r.process).map(r => ({ fault: r.process, violations: r.violations.length })),
    violations, deferred: runs.flatMap(r => (r.deferred ?? []).map(v => ({ ...v, schedule: r.schedule, seed: r.seed, fault: r.fault }))),
    failing_runs: runs.filter(runFailed), runs,
  };
}

/** The replay entries of a manifest: its failing runs, or every run of one seed. */
export function replayEntries(source: string, engine: string): RobotRun[] {
  if (/^\d+$/.test(source)) throw new Error('A bare seed replays through --seed; pass a manifest path to --replay');
  const manifest = JSON.parse(readFileSync(resolve(source), 'utf8'));
  const robot = manifest.robot ?? manifest;
  if (manifest.engine && manifest.engine !== engine) throw new Error(`Manifest ${source} was recorded on ${manifest.engine}, not ${engine}`);
  return (robot.failing_runs?.length ? robot.failing_runs : robot.runs) as RobotRun[];
}
export function runDigest(run: Pick<RobotRun, 'schedule' | 'seed' | 'fault' | 'process' | 'ops'>): string {
  return createHash('sha256').update(JSON.stringify([run.schedule, run.seed, run.fault ?? null, run.process ?? null, run.ops ?? null])).digest('hex').slice(0, 12);
}