Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.0 KB
Raw Blame
/**
 * Large-brain operational assertions (Foundations 1 F4d) for the scale tier.
 * Each runs the real CLI against the scale run's brain (or, for `sources
 * add`, a fresh managed brain) after the timed ops, and lands in the report as
 * an enforced data check plus its measurement:
 *
 *   f4d_sync_deadline      `gbrain sync` of a fresh source (1,000 files at 10k and up)
 *                          past a 1 s progress-aware deadline keeps importing and completes.
 *   f4d_embed_budget_stop  `gbrain embed --stale` stopped by its time budget exits 11
 *                          with the remaining count and the resume command (a local
 *                          stub embedding endpoint; no provider call leaves the machine).
 *   f4d_serve_boot         `gbrain serve` on the brain answers initialize and a search
 *                          inside its boot window and does not exit on the boot deadline.
 *   f4d_sources_add_20k    `gbrain sources add` registers a 20,000-file checkout on a
 *                          fresh managed brain (tiers of 20,000 pages and up only).
 */
import { execFileSync } from 'node:child_process';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import type { DataCheck } from './gates.ts';

/** Files in the sync-deadline checkout: one tenth of the brain, between 100 and 1,000 (1,000 at the 10k and 20k tiers). */
export const f4dSyncFiles = (pages: number) => Math.min(1000, Math.max(100, Math.floor(pages / 10)));
export const F4D_ADD_FILES = 20_000;
export const F4D_ADD_MIN_PAGES = 20_000;
const EMBED_BUDGET_MS = 1500;
const EMBED_STUB_DELAY_MS = 200;

export interface F4dContext {
  repo: string;
  home: string;
  pages: number;
  dim: number;
  probeToken: string;
  /** The config a fresh managed brain for the 20k-file add should use (a new PGLite path or a new Postgres database). */
  freshManagedBrain: () => Promise<{ initArgs: string[]; cleanup: () => Promise<void> }>;
}

interface Run { code: number; stdout: string; stderr: string; ms: number }

async function cli(ctx: F4dContext, args: string[], env: Record<string, string | undefined> = {}, home = ctx.home): Promise<Run> {
  const started = performance.now();
  const child = Bun.spawn([process.execPath, join(ctx.repo, 'src/cli.ts'), ...args],
    { env: { ...process.env, GBRAIN_HOME: home, ...env }, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' });
  const [stdout, stderr, code] = await Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]);
  return { code, stdout, stderr, ms: Math.round(performance.now() - started) };
}

function gitCheckout(root: string, files: number, prefix: string): void {
  for (let i = 0; i < files; i++) {
    const dir = join(root, 'notes', `batch-${String(Math.floor(i / 500)).padStart(3, '0')}`);
    if (i % 500 === 0) mkdirSync(dir, { recursive: true });
    writeFileSync(join(dir, `${prefix}-${String(i).padStart(6, '0')}.md`), `---\ntitle: ${prefix} ${i}\ntype: note\n---\n\nOperational ceiling note ${i}.\n`);
  }
  const git = (args: string[]) => execFileSync('git', ['-c', '[email protected]', '-c', 'user.name=scale', ...args], { cwd: root, stdio: 'ignore' });
  git(['init', '--quiet']);
  git(['add', '-A']);
  git(['commit', '--quiet', '-m', 'f4d fixture']);
}

/** The watchdog's extension notice: `still progressing` after a commit, or `before the run committed anything` inside the first progress window. */
export const deadlineExtended = (stderr: string) => stderr.includes('still progressing') || stderr.includes('before the run committed anything');

const tail = (run: Run) => `exit ${run.code}; stdout: ${run.stdout.slice(-600)} stderr: ${run.stderr.slice(-600)}`;
const lastJson = (text: string) => {
  const line = text.trim().split('\n').reverse().find(l => l.startsWith('{'));
  try { return line ? JSON.parse(line) as Record<string, unknown> : undefined; } catch { return undefined; }
};

async function syncDeadline(ctx: F4dContext): Promise<{ check: DataCheck; measured: Record<string, unknown> }> {
  const root = join(ctx.home, 'f4d-sync');
  const files = f4dSyncFiles(ctx.pages);
  gitCheckout(root, files, 'sync');
  const add = await cli(ctx, ['sources', 'add', 'f4d-sync', '--path', root]);
  if (add.code !== 0) return { check: { check: 'f4d_sync_deadline', status: 'fail', detail: `sources add failed: ${tail(add)}` }, measured: {} };
  const sync = await cli(ctx, ['sync', '--source', 'f4d-sync', '--no-pull', '--no-embed', '--json'],
    { GBRAIN_SYNC_MAX_RUNTIME_SECONDS: '1', GBRAIN_SYNC_STALL_ABORT_SECONDS: '300' });
  const body = lastJson(sync.stdout);
  const imported = Number(body?.added ?? 0);
  const extended = deadlineExtended(sync.stderr);
  const problem = sync.code !== 0 ? `sync exited before finishing: ${tail(sync)}`
    : imported !== files ? `sync imported ${imported} of ${files} files`
      : !extended && sync.ms > 2000 ? 'sync outlasted its 1 s deadline without the progress-aware extension notice' : null;
  return { check: { check: 'f4d_sync_deadline', status: problem ? 'fail' : 'pass', ...(problem ? { detail: problem } : {}) },
    measured: { files, deadline_s: 1, wall_ms: sync.ms, imported, extended } };
}

async function embedBudgetStop(ctx: F4dContext): Promise<{ check: DataCheck; measured: Record<string, unknown> }> {
  const vector = Array.from({ length: ctx.dim }, (_, i) => (i === 0 ? 1 : 0));
  const server = Bun.serve({ port: 0, hostname: '127.0.0.1', async fetch(request) {
    const url = new URL(request.url);
    if (url.pathname.endsWith('/models')) return Response.json({ object: 'list', data: [{ id: 'bge-m3', object: 'model' }] });
    if (!url.pathname.endsWith('/embeddings')) return new Response('not found', { status: 404 });
    const body = await request.json() as { input: string | string[] };
    const inputs = Array.isArray(body.input) ? body.input : [body.input];
    await Bun.sleep(EMBED_STUB_DELAY_MS);
    return Response.json({ object: 'list', model: 'bge-m3', usage: { prompt_tokens: inputs.length, total_tokens: inputs.length },
      data: inputs.map((_, index) => ({ object: 'embedding', index, embedding: vector })) });
  } });
  const configPath = join(ctx.home, '.gbrain', 'config.json');
  const original = readFileSync(configPath, 'utf8');
  try {
    const { embedding_disabled: _off, ...base } = JSON.parse(original) as Record<string, unknown>;
    writeFileSync(configPath, JSON.stringify({ ...base, embedding_model: 'ollama:bge-m3', embedding_dimensions: ctx.dim }, null, 2) + '\n');
    const embed = await cli(ctx, ['embed', '--stale', '--batch-size', '50', '--yes'], {
      OLLAMA_BASE_URL: `http://127.0.0.1:${server.port}/v1`, GBRAIN_EMBED_TIME_BUDGET_MS: String(EMBED_BUDGET_MS), GBRAIN_EMBED_CONCURRENCY: '1',
    });
    const verdict = `${embed.stdout}\n${embed.stderr}`.split('\n').find(l => l.includes('reason: time_budget')) ?? '';
    const left = Number(/with (\d+) stale chunk\(s\) left/.exec(verdict)?.[1] ?? NaN);
    const problem = embed.code !== 11 ? `embed exited ${embed.code}, expected the budget-stop status 11: ${tail(embed)}`
      : !(left > 0) ? `the budget-stop verdict does not name the remaining stale chunks: ${verdict || tail(embed)}`
        : !verdict.includes('gbrain embed --stale --catch-up') ? `the budget-stop verdict does not name the resume command: ${verdict}` : null;
    return { check: { check: 'f4d_embed_budget_stop', status: problem ? 'fail' : 'pass', ...(problem ? { detail: problem } : {}) },
      measured: { budget_ms: EMBED_BUDGET_MS, wall_ms: embed.ms, exit_code: embed.code, remaining_stale: Number.isFinite(left) ? left : null } };
  } finally {
    writeFileSync(configPath, original);
    server.stop(true);
  }
}

async function serveBoot(ctx: F4dContext): Promise<{ check: DataCheck; measured: Record<string, unknown> }> {
  const started = performance.now();
  const child = Bun.spawn([process.execPath, join(ctx.repo, 'src/cli.ts'), 'serve'],
    { env: { ...process.env, GBRAIN_HOME: ctx.home }, stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' });
  let stdout = '', stderr = '';
  const decoder = new TextDecoder();
  const pump = async (stream: ReadableStream<Uint8Array>, append: (s: string) => void) => { for await (const chunk of stream) append(decoder.decode(chunk)); };
  const pumps = [pump(child.stdout as ReadableStream<Uint8Array>, s => { stdout += s; }), pump(child.stderr as ReadableStream<Uint8Array>, s => { stderr += s; })];
  const send = (message: unknown) => (child.stdin as { write(s: string): unknown }).write(JSON.stringify(message) + '\n');
  const answered = async (id: number, limitMs: number) => {
    const deadline = performance.now() + limitMs;
    while (performance.now() < deadline) {
      const line = stdout.split('\n').find(l => { try { return JSON.parse(l).id === id; } catch { return false; } });
      if (line) return { ms: Math.round(performance.now() - started), body: JSON.parse(line) as Record<string, unknown> };
      if (child.exitCode !== null) return null;
      await Bun.sleep(25);
    }
    return null;
  };
  try {
    send({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'scale-f4d', version: '1' } } });
    const init = await answered(1, 180_000);
    send({ jsonrpc: '2.0', method: 'notifications/initialized' });
    send({ jsonrpc: '2.0', id: 2, method: 'tools/call', params: { name: 'search', arguments: { query: ctx.probeToken, limit: 5 } } });
    const search = await answered(2, 180_000);
    const timedOut = stderr.includes('boot did not complete');
    const problem = timedOut ? `serve exited on its boot deadline: ${stderr.slice(-800)}`
      : !init ? `serve never answered initialize (exit ${child.exitCode}): ${stderr.slice(-800)}`
        : !search || (search.body as { error?: unknown }).error ? `serve did not answer a search after boot: ${JSON.stringify(search?.body ?? null).slice(0, 400)} ${stderr.slice(-400)}`
          : child.exitCode !== null ? `serve exited ${child.exitCode} after answering` : null;
    return { check: { check: 'f4d_serve_boot', status: problem ? 'fail' : 'pass', ...(problem ? { detail: problem } : {}) },
      measured: { initialize_ms: init?.ms ?? null, first_search_ms: search?.ms ?? null } };
  } finally {
    child.kill('SIGTERM');
    const killer = setTimeout(() => child.kill('SIGKILL'), 30_000);
    await child.exited;
    clearTimeout(killer);
    await Promise.allSettled(pumps);
  }
}

async function sourcesAdd20k(ctx: F4dContext): Promise<{ check: DataCheck; measured: Record<string, unknown> }> {
  const home = join(ctx.home, 'f4d-managed');
  const root = join(home, 'checkout');
  mkdirSync(root, { recursive: true });
  const brain = await ctx.freshManagedBrain();
  try {
    const init = await cli(ctx, ['init', ...brain.initArgs, '--no-embedding'], { DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, home);
    if (init.code !== 0) return { check: { check: 'f4d_sources_add_20k', status: 'fail', detail: `init of the managed brain failed: ${tail(init)}` }, measured: {} };
    const t = performance.now();
    gitCheckout(root, F4D_ADD_FILES, 'add');
    const checkoutMs = Math.round(performance.now() - t);
    const add = await cli(ctx, ['sources', 'add', 'big', '--path', root], {}, home);
    const body = (() => { try { return JSON.parse(add.stdout.slice(add.stdout.indexOf('{'))) as Record<string, unknown>; } catch { return undefined; } })();
    const problem = add.stderr.includes('request_too_large') ? `sources add hit request_too_large: ${tail(add)}`
      : add.code !== 0 ? `sources add failed: ${tail(add)}`
        : body?.source_id !== 'big' || body.state !== 'committed' ? `sources add did not commit the source: ${add.stdout.slice(-600)}` : null;
    return { check: { check: 'f4d_sources_add_20k', status: problem ? 'fail' : 'pass', ...(problem ? { detail: problem } : {}) },
      measured: { files: F4D_ADD_FILES, checkout_ms: checkoutMs, add_ms: add.ms } };
  } finally {
    await brain.cleanup();
  }
}

/** Run every F4d assertion that applies at this tier; the scale brain must be closed (PGLite has one owner). */
export async function runF4dChecks(ctx: F4dContext): Promise<{ checks: DataCheck[]; measured: Record<string, Record<string, unknown>> }> {
  const checks: DataCheck[] = [];
  const measured: Record<string, Record<string, unknown>> = {};
  const steps: Array<[string, (c: F4dContext) => Promise<{ check: DataCheck; measured: Record<string, unknown> }>]> = [
    ['sync_deadline', syncDeadline], ['embed_budget_stop', embedBudgetStop], ['serve_boot', serveBoot],
    ...(ctx.pages >= F4D_ADD_MIN_PAGES ? [['sources_add_20k', sourcesAdd20k] as [string, typeof syncDeadline]] : []),
  ];
  for (const [name, step] of steps) {
    try {
      const result = await step(ctx);
      checks.push(result.check);
      measured[name] = result.measured;
    } catch (error) {
      checks.push({ check: `f4d_${name}`, status: 'fail', detail: `the assertion crashed: ${error instanceof Error ? error.message : String(error)}` });
    }
  }
  if (ctx.pages < F4D_ADD_MIN_PAGES) measured.sources_add_20k = { skipped: `runs at ${F4D_ADD_MIN_PAGES} pages and up` };
  return { checks, measured };
}