Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
4.8 KB
Raw Blame
/**
 * scripts/scale/watchdog.ts: the out-of-process phase watchdog ends a phase
 * whose main thread is blocked synchronously (as a PGLite WASM statement
 * blocks it) and names the phase, elapsed vs limit, the last progress line,
 * the X5 TODO and the override variable.
 */
import { describe, expect, test } from 'bun:test';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { X5_TODO_TITLE, superviseScaleRun, watchdogLimitsMs, type WatchedPhase } from '../../scripts/scale/watchdog.ts';

const HOUR = 3_600_000;
const limits = (vectors: number): Record<WatchedPhase, number> => ({ import: HOUR, extract: HOUR, vectors, budgets: HOUR });

function fixture(body: string): { dir: string; script: string } {
  const dir = mkdtempSync(join(tmpdir(), 'gbrain-scale-watchdog-'));
  const script = join(dir, 'child.ts');
  writeFileSync(script, body);
  return { dir, script };
}

describe('scale phase watchdog', () => {
  test('a phase blocking the main thread synchronously fails by name within limit + grace, and its home and process group are cleaned up', async () => {
    const { dir, script } = fixture(`
      import { spawn } from 'node:child_process';
      import { mkdirSync } from 'node:fs';
      const home = ${JSON.stringify(join(tmpdir(), 'unused'))}.replace('unused', 'gbrain-scale-watchdog-home-' + process.pid);
      mkdirSync(home, { recursive: true });
      const grandchild = spawn('sleep', ['60'], { stdio: 'ignore' });
      console.log('[scale] brain home: ' + home);
      console.log('[scale] grandchild: ' + grandchild.pid);
      console.log('[scale] phase vectors start');
      console.log('[scale] vectors batch 3/40: 500 pages in 12 ms');
      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);
    `);
    const lines: string[] = [];
    const write = process.stdout.write.bind(process.stdout);
    process.stdout.write = ((chunk: string | Uint8Array) => { lines.push(String(chunk)); return true; }) as typeof process.stdout.write;
    let code: number;
    let diagnostic = '';
    const t = performance.now();
    try {
      code = await superviseScaleRun({
        command: [process.execPath, script], pages: 20_000, limits: limits(1500), reproduce: 'bun run test:scale -- --pages 20000',
        onDiagnostic: text => { diagnostic = text; },
      });
    } finally {
      process.stdout.write = write;
    }
    const elapsed = performance.now() - t;
    try {
      expect(code).toBe(1);
      expect(elapsed).toBeLessThan(1500 + 3000);
      expect(diagnostic).toContain('FAIL phase watchdog: vectors ran');
      expect(diagnostic).toContain('limit 2 s');
      expect(diagnostic).toContain('last progress line: [scale] vectors batch 3/40: 500 pages in 12 ms');
      expect(diagnostic).toContain(`TODOS.md "${X5_TODO_TITLE}" (X5)`);
      expect(readFileSync(join(import.meta.dir, '../../TODOS.md'), 'utf8')).toContain(X5_TODO_TITLE);
      expect(diagnostic).toContain('GBRAIN_SCALE_PHASE_LIMIT_MS_VECTORS=<ms>');
      expect(diagnostic).toContain('bun run test:scale -- --pages 20000');
      const output = lines.join('');
      const home = /\[scale\] brain home: (.+)/.exec(output)![1]!;
      expect(existsSync(home)).toBe(false);
      const grandchild = Number(/\[scale\] grandchild: (\d+)/.exec(output)![1]);
      await Bun.sleep(100);
      expect(() => process.kill(grandchild, 0)).toThrow();
    } finally {
      rmSync(dir, { recursive: true, force: true });
    }
  });

  test('a phase that finishes inside its limit passes the child exit code through and leaves its home alone', async () => {
    const home = mkdtempSync(join(tmpdir(), 'gbrain-scale-watchdog-keep-'));
    const { dir, script } = fixture(`
      console.log('[scale] brain home: ${home}');
      console.log('[scale] phase vectors start');
      console.log('[scale] phase vectors done in 1 ms');
      console.log('[scale] phase ops start');
      process.exit(7);
    `);
    try {
      expect(await superviseScaleRun({ command: [process.execPath, script], pages: 20_000, limits: limits(60_000), reproduce: 'r' })).toBe(7);
      expect(existsSync(home)).toBe(true);
    } finally {
      rmSync(dir, { recursive: true, force: true });
      rmSync(home, { recursive: true, force: true });
    }
  });

  test('limits are 1.5x the gate ceilings, and the override variable replaces one phase', () => {
    const at20k = watchdogLimitsMs(20_000, {});
    expect(at20k).toEqual({ import: 45 * 60_000, extract: 22.5 * 60_000, vectors: 11.25 * 60_000, budgets: 15 * 60_000 });
    expect(watchdogLimitsMs(20_000, { GBRAIN_SCALE_PHASE_LIMIT_MS_VECTORS: '5000' }).vectors).toBe(5000);
    expect(() => watchdogLimitsMs(20_000, { GBRAIN_SCALE_PHASE_LIMIT_MS_EXTRACT: 'soon' })).toThrow('GBRAIN_SCALE_PHASE_LIMIT_MS_EXTRACT must be a positive number');
  });
});