Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
6.6 KB
Raw Blame
/**
 * Out-of-process phase watchdog for the scale harness (A6/X5; ENG-2, DX-13).
 *
 * scripts/scale/run.ts re-runs itself as a child under this supervisor. A
 * PGLite statement runs synchronously inside WASM on the child's main thread,
 * so an in-process timer cannot fire during a stall; the parent reads the
 * child's `[scale] phase <name> start` lines, times each watched phase, and
 * SIGKILLs the child's process group at the limit. The diagnostic names the
 * phase, elapsed vs limit, the last progress line, the X5 TODO and the
 * override variable, then the supervisor removes the child's brain home and
 * scale database (the killed child cannot run its own cleanup).
 *
 * Watched phases and the child phases they cover:
 *   import   corpus, schema, import_files, extract, vectors (the gate's import window)
 *   extract  extract
 *   vectors  vectors
 *   budgets  ops, reimport, cold_query, writers (the gate's budgets window)
 * Limits are 1.5x the gate ceilings (scripts/scale/gates.ts phaseLimitsMs), so
 * the gate stays the verdict and the watchdog only ends hangs. Override one
 * with GBRAIN_SCALE_PHASE_LIMIT_MS_<PHASE> (e.g. ..._VECTORS=3600000) for a
 * local investigation.
 */
import { spawn } from 'node:child_process';
import { rmSync } from 'node:fs';
import { phaseLimitsMs } from './gates.ts';

export type WatchedPhase = 'import' | 'extract' | 'vectors' | 'budgets';
export const WATCHED_PHASES: WatchedPhase[] = ['import', 'extract', 'vectors', 'budgets'];
const COVERS: Record<string, WatchedPhase[]> = {
  corpus: ['import'], schema: ['import'], import_files: ['import'],
  extract: ['import', 'extract'], vectors: ['import', 'vectors'],
  ops: ['budgets'], reimport: ['budgets'], cold_query: ['budgets'], writers: ['budgets'],
};
export const WATCHDOG_GRACE = 1.5;
export const X5_TODO_TITLE = 'PGLite bulk-embedding cost at 20k+ chunks';
export const X5_TODO = `TODOS.md "${X5_TODO_TITLE}" (X5)`;
export const HOME_LINE = '[scale] brain home: ';
export const DATABASE_LINE = '[scale] scale database: ';

export const overrideVariable = (phase: WatchedPhase) => `GBRAIN_SCALE_PHASE_LIMIT_MS_${phase.toUpperCase()}`;

export function watchdogLimitsMs(pages: number, env: Record<string, string | undefined> = process.env): Record<WatchedPhase, number> {
  const gate = phaseLimitsMs(pages);
  const limits: Record<WatchedPhase, number> = {
    import: gate.import * WATCHDOG_GRACE,
    extract: gate.import * 0.5 * WATCHDOG_GRACE,
    vectors: gate.import * 0.25 * WATCHDOG_GRACE,
    budgets: gate.budgets * WATCHDOG_GRACE,
  };
  for (const phase of WATCHED_PHASES) {
    const raw = env[overrideVariable(phase)];
    if (raw === undefined || raw === '') continue;
    const ms = Number(raw);
    if (!Number.isFinite(ms) || ms <= 0) throw new Error(`${overrideVariable(phase)} must be a positive number of milliseconds, got '${raw}'.`);
    limits[phase] = ms;
  }
  return limits;
}

export function watchdogDiagnostic(phase: WatchedPhase, elapsedMs: number, limitMs: number, lastLine: string, reproduce: string): string {
  return [
    `[scale] FAIL phase watchdog: ${phase} ran ${Math.round(elapsedMs / 1000)} s, limit ${Math.round(limitMs / 1000)} s; the harness was killed (exit 1).`,
    `[scale]   last progress line: ${lastLine || '(none)'}`,
    `[scale]   Why: a phase past its watchdog limit (${WATCHDOG_GRACE}x its gate ceiling unless overridden) is a stall, and an in-process timer cannot fire while PGLite blocks the main thread.`,
    `[scale]   Known issue: ${X5_TODO} tracks PGLite bulk-embedding cost and the 20k vectors stall history.`,
    `[scale]   Fix: profile the named phase; to let it run longer locally, set ${overrideVariable(phase)}=<ms> and rerun: ${reproduce}`,
  ].join('\n');
}

export interface SuperviseOptions {
  command: string[];
  pages: number;
  reproduce: string;
  env?: Record<string, string | undefined>;
  limits?: Record<WatchedPhase, number>;
  /** Drops the child's scale database after a kill; run.ts passes a Postgres admin connection. */
  dropDatabase?: (name: string) => Promise<void>;
  onDiagnostic?: (text: string) => void;
}

/** Run the harness child under the watchdog; resolves to the exit code to use. */
export async function superviseScaleRun(opts: SuperviseOptions): Promise<number> {
  const limits = opts.limits ?? watchdogLimitsMs(opts.pages, opts.env);
  const child = spawn(opts.command[0]!, opts.command.slice(1), {
    env: { ...(opts.env ?? process.env), GBRAIN_SCALE_SUPERVISED: '1' },
    stdio: ['ignore', 'pipe', 'pipe'],
    detached: true,
  });
  const started = new Map<WatchedPhase, number>();
  let lastLine = '';
  let home: string | undefined;
  let database: string | undefined;
  let fired: string | undefined;

  const consume = (stream: NodeJS.ReadableStream, sink: NodeJS.WriteStream) => {
    let buffered = '';
    stream.on('data', (chunk: Buffer) => {
      sink.write(chunk);
      buffered += chunk.toString();
      const lines = buffered.split(/\r?\n/);
      buffered = lines.pop() ?? '';
      for (const raw of lines) {
        const line = raw.trim();
        if (!line) continue;
        lastLine = line.slice(0, 500);
        if (line.startsWith(HOME_LINE)) home = line.slice(HOME_LINE.length);
        if (line.startsWith(DATABASE_LINE)) database = line.slice(DATABASE_LINE.length);
        const phase = /^\[scale\] phase (\w+) start$/.exec(line)?.[1];
        const now = performance.now();
        for (const p of WATCHED_PHASES) if (started.has(p) && !(COVERS[phase ?? '']?.includes(p) ?? true)) started.delete(p);
        for (const p of COVERS[phase ?? ''] ?? []) if (!started.has(p)) started.set(p, now);
      }
    });
  };
  consume(child.stdout!, process.stdout);
  consume(child.stderr!, process.stderr);

  const exited = new Promise<number>(resolve => child.on('exit', (code, signal) => resolve(code ?? (signal ? 1 : 3))));
  const timer = setInterval(() => {
    if (fired) return;
    const now = performance.now();
    for (const [phase, since] of started) {
      if (now - since <= limits[phase]) continue;
      fired = watchdogDiagnostic(phase, now - since, limits[phase], lastLine, opts.reproduce);
      try { process.kill(-child.pid!, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
      return;
    }
  }, 250);
  const code = await exited;
  clearInterval(timer);
  if (!fired) return code;
  console.log(fired);
  opts.onDiagnostic?.(fired);
  if (home) rmSync(home, { recursive: true, force: true });
  if (database && opts.dropDatabase) {
    await opts.dropDatabase(database).catch(e => console.log(`[scale] could not drop ${database}: ${e instanceof Error ? e.message : String(e)}; drop it by hand with DROP DATABASE ${database} WITH (FORCE)`));
  }
  return 1;
}