Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.4 KB
Raw Blame
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { availableParallelism, loadavg, tmpdir, totalmem } from 'node:os';
import { basename, dirname, join, resolve } from 'node:path';
import postgres from '#postgres'
import { assertSafeE2eDatabaseUrl } from '../../test/helpers/db-guard.ts';
import { distribution, type HarnessConfig } from './harness.ts';
import { keylessBrainEnv } from '../../test/helpers/provider-env.ts';
import { boundedDiagnostic, connectionDiagnostic, diagnosticError, retentionMetadata } from './failure-diagnostics.ts';
import { explorationSeeds, replayEntries, runDigest, runRobotPhase, type RobotRun, type RobotScope } from './robot-driver.ts';
import { shrinkRun } from './shrink.ts';

export interface ValidationOptions {
  engine: 'pglite' | 'postgres'; schedules?: number; operations?: number; seed?: number;
  crashes?: boolean; databaseUrl?: string; manifest?: string;
  /** Crash-robot ceiling in seconds: a run past it fails; omitted or 0 skips the phase (the CLI defaults to 1200 on PGLite, 880 on Postgres). */
  robotSeconds?: number;
  /** `pr` skips the Postgres lease-bound seams; `full` (default) crashes every seam. */
  robotScope?: RobotScope;
  /** A UTC day number: the robot also runs that day's explorationSeeds (the nightly). */
  exploreDay?: number;
  /** Re-run a recorded manifest's failing crash-robot runs (and nothing else). */
  replay?: string;
  /** Shrink the first failing crash-robot run of a manifest to a minimal reproducing sequence. */
  shrink?: string;
}
interface Event { event: string; [key: string]: any; }
export const CRASH_BOUNDARIES = ['admitted', 'prepared', 'before_publication', 'staging_flushed',
  'after_publication', 'before_commit', 'after_commit', 'after_response'] as const;
export function childEnvironment(home: string): Record<string, string> {
  // Preserve the runtime executable/search paths, never an operator's brain or provider configuration.
  const env = Object.fromEntries(Object.entries(process.env).filter(([key, value]) => value !== undefined &&
    !/^(GBRAIN_|CONDUCTOR_|MCP_|OPENCLAW_|ANTHROPIC_|OPENAI_|DATABASE_URL$)/.test(key))) as Record<string, string>;
  return keylessBrainEnv(env, home, { GBRAIN_CI_DISABLE_TEST_ENV_FILE: '1', GBRAIN_PERSISTENCE_FIXTURE_HOME: home });
}
export function spawnWorker(configPath: string, home: string, role: string, args: string[] = [], environment: Record<string, string> = {}) {
  const child = Bun.spawn([process.execPath, '--no-env-file', resolve(import.meta.dir, 'worker.ts'), role, configPath, ...args],
    { env: { ...childEnvironment(home), ...environment }, stdin: 'ignore', stdout: 'pipe', stderr: 'inherit' });
  const events: Event[] = []; const waiters = new Set<() => void>(); let ended = false; let tail = '';
  const reading = (async () => {
    const decoder = new TextDecoder(); let pending = '';
    for await (const chunk of child.stdout) {
      pending += decoder.decode(chunk, { stream: true });
      for (;;) { const newline = pending.indexOf('\n'); if (newline < 0) break;
        const line = pending.slice(0, newline); pending = pending.slice(newline + 1); tail = `${tail}\n${line}`.slice(-16000);
        try { const event = JSON.parse(line); if (typeof event.event === 'string') events.push(event); } catch { /* driver startup diagnostics */ }
        for (const wake of waiters) wake();
      }
    }
    ended = true; for (const wake of waiters) wake();
  })();
  async function event(name: string, timeoutMs = 5_400_000): Promise<Event> {
    const deadline = Date.now() + timeoutMs;
    for (;;) {
      const index = events.findIndex(e => e.event === name); if (index >= 0) return events.splice(index, 1)[0];
      const failure = events.find(e => e.event === 'failure'); if (failure) throw new Error(`${role}: ${failure.message}`);
      if (ended) throw new Error(`${role} exited before ${name} (exit=${await child.exited}): ${tail}`);
      assert(Date.now() < deadline, `${role} timed out before ${name}`);
      await new Promise<void>(done => {
        const timer = setTimeout(wake, Math.min(1000, deadline - Date.now()));
        function wake() { clearTimeout(timer); waiters.delete(wake); done(); } waiters.add(wake);
      });
    }
  }
  async function done(): Promise<Event> { const result = await event('done'); assert.equal(await child.exited, 0, `${role} failed`); await reading; return result; }
  return { child, event, done,
    failureDiagnostics: () => events.filter(event => event.event === 'failure' && event.diagnostics).slice(0, 1)
      .map(event => ({ role, pid: child.pid, ...event.diagnostics })),
    async kill() { if (child.exitCode === null) child.kill('SIGKILL'); await child.exited; await reading; } };
}

/** A full robot gate ran the whole budget, skipped nothing, and every requested crash happened and recovered cleanly (#6584). */
export function robotFullGate(robot: Pick<Awaited<ReturnType<typeof runRobotPhase>>, 'violations' | 'fault_hook_errors' | 'faults_not_reached'>
  & { lease_bound_seams_skipped?: string[]; lease_bound_faults_skipped?: string[] }, scope: RobotScope): boolean {
  return scope === 'full' && robot.violations.length === 0
    && robot.fault_hook_errors.length === 0 && robot.faults_not_reached.length === 0
    && (robot.lease_bound_seams_skipped ?? []).length === 0 && (robot.lease_bound_faults_skipped ?? []).length === 0;
}

/** A run past its ceiling: the fixed schedule list no longer fits this runner class, so the gate fails instead of covering less. */
export function robotOverBudget(durationMs: number, seconds: number): string {
  return `Crash robot over its ceiling: ran ${Math.round(durationMs / 1000)} s against --robot-seconds=${seconds}. Why: every run executes the same `
    + 'schedule list (scripts/persistence/robot-driver.ts ROBOT_RANDOM_SEEDS) so a passed run of a tree can be reused; past the ceiling the list '
    + 'no longer fits this runner class. Fix: find the slow runs in the manifest; speed them up, or resize the list and the ceiling together.';
}

/** One line per robot cell for the job summary: elapsed time as a share of the ceiling, so drift shows long before it fails. */
export function robotCeilingUse(engine: string, scope: RobotScope, durationMs: number, seconds: number): string {
  const elapsed = Math.round(durationMs / 1000);
  return `Crash robot ${engine} (${scope}): ${elapsed} s of its ${seconds} s ceiling (${Math.round(100 * durationMs / (seconds * 1000))}%)\n`;
}

/** The failing runs plus the exact command that replays them and the one that shrinks the first. */
export function robotFailure(failing: RobotRun[], options: ValidationOptions): string {
  const manifest = options.manifest ?? `.context/persistence-${options.engine}-manifest.json`;
  const env = options.engine === 'postgres' ? 'DATABASE_URL=<test url> ' : '';
  const lines = failing.slice(0, 5).map(run => `  ${run.schedule} seed=${run.seed} ${run.fault ? `${run.fault.point}#${run.fault.nth}` : run.process ?? 'no crash'} [${runDigest(run)}]: `
    + (run.violations.length ? run.violations.slice(0, 3).map(v => `${v.class}: ${v.detail}`).join(' | ')
      : run.hook_error !== undefined ? `fault_hook_error: ${run.hook_error}` : 'fault_not_reached: the worker finished without reaching the requested fault'));
  return [`Crash robot found ${failing.length} failing run(s):`, ...lines,
    `Replay: ${env}bun --no-env-file scripts/persistence/validate.ts --engine=${options.engine} --replay=${manifest}`,
    `Shrink: ${env}bun --no-env-file scripts/persistence/validate.ts --engine=${options.engine} --shrink=${manifest} (writes ${manifest}.shrunk.json)`].join('\n');
}

/** Every PostgreSQL phase owns a new database; disk PGLite always runs in children. */
export async function runValidation(options: ValidationOptions) {
  const counts = { schedules: options.schedules ?? 1000, operations: options.operations ?? 10_000 };
  for (const [key, value] of Object.entries(counts)) assert(Number.isSafeInteger(value) && value >= 0, `Invalid ${key}`);
  assert(Number.isSafeInteger(options.seed ?? 5105) && (options.seed ?? 5105) >= 0 && (options.seed ?? 5105) <= 0xFFFFFFFF, 'Invalid unsigned 32-bit seed');
  const scratch = mkdtempSync(join(tmpdir(), 'gbrain-persistence-validation-')); const home = join(scratch, 'home'); mkdirSync(home);
  const children: ReturnType<typeof spawnWorker>[] = []; const databases: string[] = [];
  const ownerUrls: string[] = [];
  let admin: ReturnType<typeof postgres> | undefined;
  let originalFailure = false;
  const manifest: Record<string, any> = { version: 1, engine: options.engine, runtime: `bun-${Bun.version}`,
    platform: process.platform, architecture: process.arch, seed: options.seed ?? 5105,
    environment: { logical_cpus: availableParallelism(), memory_bytes: totalmem(), load_average_at_start: loadavg() },
    started_at: new Date().toISOString(), status: 'running', requested: { ...counts, crashes: options.crashes !== false },
    managed_persistence: true, scope: 'activated real journal/coordinator; fixture-only loopback transport for PGLite producers', crash_cases: [], phase_inputs: {} };
  const at = performance.now();
  try {
    if (options.engine === 'postgres') {
      assert(options.databaseUrl, 'Postgres validation requires an explicit test DATABASE_URL');
      assertSafeE2eDatabaseUrl(options.databaseUrl); admin = postgres(options.databaseUrl, { max: 1, onnotice() {} });
    }
    async function phase(name: string): Promise<{ config: HarnessConfig; path: string }> {
      const root = join(scratch, name); mkdirSync(root);
      manifest.phase_inputs[name] = Object.fromEntries(['scripts/persistence/harness.ts', 'scripts/persistence/schedules.ts',
        'scripts/persistence/worker.ts', 'scripts/persistence/validate.ts', 'scripts/persistence/failure-diagnostics.ts',
        'src/core/atomic-write.ts', 'src/core/persistence/staging.ts', 'src/core/persistence/model.ts',
        'src/core/persistence/effect-recovery.ts', 'src/core/persistence/effect-model.ts', 'src/core/persistence/effects.ts',
        'src/core/persistence/coordinator.ts', 'src/core/persistence/consumer.ts',
        'src/core/persistence/journal.ts', 'src/core/persistence/activation.ts', 'src/core/persistence/filesystem-guard.ts',
        'src/core/persistence/identity.ts', 'src/core/persistence/ownership.ts', 'src/core/pglite-engine.ts', 'src/core/postgres-engine.ts',
        'src/core/persistence/fault-points.ts', 'scripts/persistence/ops.ts', 'scripts/persistence/model.ts', 'scripts/persistence/generator.ts',
        'scripts/persistence/crash-robot.ts', 'scripts/persistence/robot-driver.ts', 'scripts/persistence/shrink.ts'].map(file =>
        [file, createHash('sha256').update(readFileSync(resolve(import.meta.dir, '../..', file))).digest('hex')]));
      let databaseUrl: string | undefined;
      if (admin) {
        const database = `gbrain_persistence_test_${randomUUID().replaceAll('-', '')}`;
        await admin.unsafe(`CREATE DATABASE ${database}`); databases.push(database);
        const url = new URL(options.databaseUrl!); url.pathname = `/${database}`; databaseUrl = url.toString();
      }
      const config: HarnessConfig = { kind: options.engine, root, dataDir: join(root, 'data'), databaseUrl,
        hostId: randomUUID(), seed: options.seed ?? 5105, ...counts,
        sourceIds: Array.from({ length: 4 }, (_, i) => `persistence-test-${i}`), principalIds: Array.from({ length: 4 }, () => randomUUID()) };
      const path = join(root, 'config.json'); writeFileSync(path, JSON.stringify(config), { mode: 0o600 }); return { config, path };
    }
    function start(path: string, role: string, ...args: string[]) { const child = spawnWorker(path, home, role, args); children.push(child); return child; }
    const robotOptions = (extra: Partial<Parameters<typeof runRobotPhase>[0]> = {}) => ({ engine: options.engine, seed: options.seed ?? 5105,
      seconds: options.robotSeconds ?? 0, scope: options.robotScope ?? 'full',
      extraSeeds: options.exploreDay === undefined ? [] : explorationSeeds(options.exploreDay), scratch, home, admin, databaseUrl: options.databaseUrl, databases,
      pooledUrl: options.engine === 'postgres' ? process.env.GBRAIN_PGBOUNCER_URL || undefined : undefined,
      spawn: spawnWorker, track: (child: ReturnType<typeof spawnWorker>) => { children.push(child); }, ...extra });
    if (options.replay || options.shrink) {
      const entries = replayEntries((options.replay ?? options.shrink)!, options.engine);
      if (options.shrink) {
        manifest.shrink = await shrinkRun(entries.find(e => e.violations.length) ?? entries[0], (run, keep) => runRobotPhase(robotOptions({ replay: [run], ...(keep ? { keep } : {}) })));
        const shrunkPath = `${resolve(options.shrink)}.shrunk.json`;
        writeFileSync(shrunkPath, `${JSON.stringify({ engine: options.engine, reproduced: `${manifest.shrink.reproduced}/${manifest.shrink.reruns}`,
          failing_runs: [manifest.shrink.shrunk] }, null, 2)}\n`);
        process.stderr.write(`[persistence] shrunk ${manifest.shrink.original.ops} ops to ${manifest.shrink.shrunk.ops?.length}; reproduced ${manifest.shrink.reproduced}/3; ${shrunkPath}\n`
          + `[persistence] Replay: bun --no-env-file scripts/persistence/validate.ts --engine=${options.engine} --replay=${shrunkPath}\n`);
        manifest.status = manifest.shrink.accepted ? 'passed' : 'failed'; manifest.full_gate = false; return manifest;
      }
      manifest.robot = await runRobotPhase(robotOptions({ replay: entries }));
      manifest.status = manifest.robot.failing_runs.length ? 'failed' : 'passed'; manifest.full_gate = false;
      if (manifest.robot.failing_runs.length) throw new Error(robotFailure(manifest.robot.failing_runs, options));
      return manifest;
    }
    if (options.robotSeconds) {
      manifest.robot = await runRobotPhase(robotOptions());
      process.stderr.write(`[persistence] ${options.engine}: crash robot ran ${manifest.robot.sequences_x_crash_points} sequence x crash point runs over ${manifest.robot.schedules} schedules\n`);
      const use = robotCeilingUse(options.engine, options.robotScope ?? 'full', manifest.robot.duration_ms, options.robotSeconds);
      process.stderr.write(`[persistence] ${use}`);
      if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, use);
      if (manifest.robot.failing_runs.length) throw new Error(robotFailure(manifest.robot.failing_runs, options));
      if (manifest.robot.over_budget) throw new Error(robotOverBudget(manifest.robot.duration_ms, options.robotSeconds));
    }
    if (options.crashes !== false) for (const boundary of CRASH_BOUNDARIES) {
      const { path } = await phase(`crash-${boundary}`); const requestId = randomUUID();
      const child = start(path, 'crash', boundary, requestId);
      const reached = await child.event(boundary === 'after_response' ? 'response_ready' : 'boundary');
      assert.equal(reached.boundary, boundary); assert.equal(reached.requestId, requestId);
      if (boundary === 'after_response') {
        const response = await fetch(reached.url, { signal: AbortSignal.timeout(5_000) });
        assert(response.ok); const receipt = await response.json();
        assert.equal(receipt.request_id, requestId); assert.equal(receipt.state, 'committed');
        assert.equal(receipt.persistence.mode, 'filesystem');
      }
      await child.kill();
      const recovered = await start(path, 'recover', boundary, requestId).done();
      assert.equal(recovered.result.boundary, boundary);
      manifest.crash_cases.push({ ...recovered.result, ...(boundary === 'after_response' ? { response_read_before_kill: true } : {}) });
      process.stderr.write(`[persistence] ${options.engine}: SIGKILL/${boundary} durable recovery verified\n`);
    }
    if (counts.schedules) {
      const { path } = await phase('schedules'); await start(path, 'initialize').done();
      manifest.schedules = (await start(path, 'schedules').done()).result;
    }
    if (counts.operations) {
      const { path } = await phase('soak'); await start(path, 'initialize').done(); const started = performance.now();
      const owners = Array.from({ length: options.engine === 'postgres' ? 2 : 1 }, () => start(path, 'owner'));
      const ready = await Promise.all(owners.map(owner => owner.event('ready')));
      ownerUrls.push(...ready.map(owner => owner.url));
      const results = await Promise.all(Array.from({ length: 4 }, (_, i) => start(path, 'producer', String(i), ready[0].url).done()));
      const ownerResults = await Promise.all(ready.map(async owner => {
        const response = await fetch(new URL('stop', owner.url), { method: 'POST' }); assert(response.ok); return response.json();
      }));
      for (const owner of owners) assert.equal(await owner.child.exited, 0, 'resident must drain and close successfully');
      for (const owner of ownerResults) assert.deepEqual(owner.errors, [], 'resident storage errors require investigation');
      const verification = (await start(path, 'verify-soak').done()).result;
      assert.equal(results.reduce((sum, row) => sum + row.result.completed, 0), counts.operations);
      const duration = performance.now() - started;
      manifest.soak = { ...verification, producer_processes: 4, owner_processes: owners.length,
        producer_admission: options.engine === 'postgres' ? 'independent database clients' : 'resident fixture loopback endpoint',
        duration_ms: duration, operations_per_second: counts.operations / (duration / 1000),
        duplicate_replays: results.reduce((sum, row) => sum + row.result.replays, 0),
        admission: distribution(results.flatMap(row => row.result.admission_ms)),
        caller_completion: distribution(results.flatMap(row => row.result.completion_ms)),
        concurrent_canonical_read: distribution(ownerResults.flatMap(owner => owner.concurrent_read_ms)),
        peak_owner_rss_bytes: ownerResults.reduce((peak: number | null, owner) =>
          owner.peak_rss_bytes === null ? peak : Math.max(peak ?? 0, owner.peak_rss_bytes), null) };
    }
    manifest.status = 'passed';
    const executedBoundaries = manifest.crash_cases.map((entry: { boundary: string }) => entry.boundary);
    if (options.crashes !== false) assert.deepEqual(executedBoundaries, [...CRASH_BOUNDARIES]);
    if (manifest.robot) manifest.robot_full_gate = robotFullGate(manifest.robot, options.robotScope ?? 'full');
    manifest.full_gate = counts.schedules >= 1000 && counts.operations >= 10_000
      && executedBoundaries.length === CRASH_BOUNDARIES.length && CRASH_BOUNDARIES.every((boundary, index) => executedBoundaries[index] === boundary)
      && manifest.crash_cases.every((entry: { staging_cleanup_verified?: boolean }) => entry.staging_cleanup_verified === true)
      && manifest.crash_cases.find((entry: { boundary: string }) => entry.boundary === 'staging_flushed')?.flushed_before_rename_verified === true
      && manifest.crash_cases.find((entry: { boundary: string }) => entry.boundary === 'staging_flushed')?.unexpected_staging_preserved === true
      && manifest.crash_cases.find((entry: { boundary: string }) => entry.boundary === 'after_response')?.response_read_before_kill === true;
    return manifest;
  } catch (error) {
    originalFailure = true; manifest.status = 'failed'; manifest.full_gate = false; manifest.failure = String(error);
    // Cached producer state is already available; a stuck owner adds only a
    // bounded timeout marker. URLs and credentials never enter this manifest.
    manifest.failure_diagnostics = await boundedDiagnostic(async () => ({
      workers: children.flatMap(child => child.failureDiagnostics()),
      owners: await Promise.all(ownerUrls.map(url => boundedDiagnostic(async () => {
        const response = await fetch(new URL('diagnostics', url), { signal: AbortSignal.timeout(1_500) });
        assert(response.ok, 'Owner diagnostic endpoint failed'); return response.json();
      }, 1_750))),
    }));
    if (admin) manifest.connection_diagnostic = await boundedDiagnostic(() => connectionDiagnostic(admin!));
    throw error;
  }
  finally {
    const stopChildren = async () => {
      const stopped = await Promise.allSettled(children.map(child => child.kill()));
      const failed = stopped.find(result => result.status === 'rejected');
      if (failed?.status === 'rejected') throw failed.reason;
    };
    const shutdown = originalFailure ? await boundedDiagnostic(stopChildren) : (await stopChildren(), { status: 'ok' as const });
    if (admin && !originalFailure) for (const database of databases) await admin.unsafe(`DROP DATABASE IF EXISTS ${database} WITH (FORCE)`);
    if (admin) {
      if (originalFailure) manifest.admin_close_diagnostic = await boundedDiagnostic(() => admin!.end({ timeout: 1 }));
      else await admin.end();
    }
    if (manifest.status === 'failed') {
      const retainedPath = options.manifest ? `${resolve(options.manifest)}.retained.json` : join(scratch, 'retained.json');
      try {
        mkdirSync(dirname(retainedPath), { recursive: true });
        writeFileSync(retainedPath, `${JSON.stringify({ ...retentionMetadata(scratch, databases),
          worker_pids: children.map(child => child.child.pid), worker_shutdown: shutdown }, null, 2)}\n`, { mode: 0o600 });
        manifest.failure_artifacts = { retained: true, metadata_file: basename(retainedPath) };
        process.stderr.write(`[persistence] Failed fixtures retained. Private cleanup metadata: ${retainedPath}\n`);
      } catch (error) {
        manifest.failure_artifacts = { retained: true, metadata_error_code: diagnosticError(error) };
        // The scratch directory survives even when the report volume is full.
        process.stderr.write(`[persistence] Could not write cleanup metadata; synthetic scratch retained at ${scratch}\n`);
      }
    }
    manifest.finished_at = new Date().toISOString(); manifest.duration_ms = performance.now() - at;
    try {
      if (options.manifest) { mkdirSync(dirname(resolve(options.manifest)), { recursive: true }); writeFileSync(options.manifest, `${JSON.stringify(manifest, null, 2)}\n`); }
    } catch (error) {
      if (!originalFailure) throw error;
      process.stderr.write(`[persistence] Manifest write failed (${diagnosticError(error)}); original failure preserved.\n`);
    }
    if (!originalFailure) rmSync(scratch, { recursive: true, force: true });
  }
}
if (import.meta.main) {
  const args = new Map(process.argv.slice(2).map(arg => { const [key, ...value] = arg.replace(/^--/, '').split('='); return [key, value.join('=')]; }));
  for (const key of args.keys()) assert(['engine', 'schedules', 'operations', 'seed', 'no-crashes', 'manifest', 'robot-seconds', 'robot-scope', 'explore-day', 'replay', 'shrink'].includes(key), `Unknown option: ${key}`);
  const engine = args.get('engine') ?? 'pglite'; assert(engine === 'pglite' || engine === 'postgres');
  assert(['pr', 'full'].includes(args.get('robot-scope') ?? 'full'), '--robot-scope is pr or full');
  const result = await runValidation({ engine, schedules: Number(args.get('schedules') ?? 1000), operations: Number(args.get('operations') ?? 10_000),
    seed: Number(args.get('seed') ?? 5105), crashes: !args.has('no-crashes'), databaseUrl: process.env.DATABASE_URL,
    robotSeconds: Number(args.get('robot-seconds') ?? (engine === 'postgres' ? 880 : 1200)),
    robotScope: (args.get('robot-scope') ?? 'full') as RobotScope, exploreDay: args.has('explore-day') ? Number(args.get('explore-day')) : undefined, replay: args.get('replay') || undefined, shrink: args.get('shrink') || undefined,
    manifest: args.get('manifest') ?? `.context/persistence-${engine}-manifest.json` });
  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
}