Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Gate policy for the scale tier (spec F4c; O-CEO-16 / O-ENG-16 / O-CEO-9).
* Pure: `evaluateScaleGates` reads a finished report and decides which gates
* pass, which fail, and the exit code. scripts/scale/run.ts calls it, the
* scale-tier workflow enforces it with `--enforce`, and
* test/scripts/scale-gates.test.ts plants slow ops and wrong answers in it.
*
* Enforced from day one (under --enforce): import rate, known answers for
* every timed op and data check, no-op re-import, no duplicate documents
* across sources, the import phase timer.
* The stats-dependent gates (planner health and the budgets phase timer) are
* enforced only when PLANNER_HEALTH_ENFORCED is true.
* Interactive ceilings and calibrated budgets stay report-only until
* scripts/scale/trend.ts says "ceilings stable" and a reviewer sets the repo
* variable GBRAIN_SCALE_ENFORCE_CEILINGS=1.
*/
import { ORPHANS_MAX_LIMIT } from '../../src/core/ops/orphans.ts';
import { PLANNER_STATS_MIN_PENDING } from '../../src/core/planner-stats.ts';
/**
* The one switch for the stats-dependent gates: planner health (hot-table
* statistics after the first read, Nested Loop inner loops in the key plans)
* and the budgets phase timer, whose time un-analyzed plans dominate. On since
* F4b (row-delta ANALYZE during import and on the first planner-sensitive
* read): at 10k PGLite pages its key plans run 42-52 inner loops against a
* 100k gate and the budgets phase 25 s against 300 s. Set false to make them
* report-only again.
*/
export const PLANNER_HEALTH_ENFORCED = true;
export const RATE_RATIO_MAX = 1.5;
/** Below this size per-process warmup dominates the per-page cost, so the rate gate is reported but not enforced. */
export const RATE_MIN_PAGES = 1000;
export const TOTAL_VS_HALF_MAX = 2.5;
/**
* One timing basis of the import: per-page cost of the first and last 10%, and total vs the halfway mark.
* The two ratios leave out the one stall `excusedStall` sets aside; `total_ms` and `ms_at_half` are the measured sums.
*/
export interface RateMeasure {
per_page_ms_first10: number;
per_page_ms_last10: number;
rate_ratio: number;
total_ms: number;
ms_at_half: number;
total_vs_half: number;
excused?: ExcusedStall;
}
/** The milliseconds the rate measure set aside, and the 1-based pages of the slice that held them. */
export interface ExcusedStall { ms: number; first_page: number; last_page: number }
/**
* Per-file wall and CPU milliseconds from `gbrain import --progress-json --progress-interval 0`
* stderr: one `import.files` tick per file, each with cumulative `elapsed_ms` and `cpu_ms`.
* A file without a tick, or a tick without a field, yields NaN for the caller to reject.
*/
export function importProgressPerFile(stderr: string): { wallMs: number[]; cpuMs: number[] } {
const wall: number[] = [];
const cpu: number[] = [];
for (const line of stderr.split('\n')) {
if (!line.startsWith('{')) continue;
const event = JSON.parse(line) as { event?: string; phase?: string; done?: number; elapsed_ms?: number; cpu_ms?: number };
if (event.event !== 'tick' || event.phase !== 'import.files' || typeof event.done !== 'number') continue;
wall[event.done - 1] = event.elapsed_ms ?? Number.NaN;
cpu[event.done - 1] = event.cpu_ms ?? Number.NaN;
}
const perFile = (cumulative: number[]) => Array.from(cumulative, (ms, i) => (ms ?? Number.NaN) - (i > 0 ? cumulative[i - 1] ?? Number.NaN : 0));
return { wallMs: perFile(wall), cpuMs: perFile(cpu) };
}
/**
* #7939: one stall of the import (a paused database container, a WAL checkpoint, a burst from another tenant of a
* shared CI host) lands in one page's time and once carried a whole tenth's mean past the gate on its own. Work that
* grows with the brain does not look like that: it recurs. So the measure sets aside at most one stall per import,
* and only the part of it no other part of the import repeats:
*
* - Each tenth of the import is cut into ten slices. A slice's excess is its mean per-page cost above the median
* slice of its tenth. When the second-largest excess anywhere in the import is over a quarter of the largest, the
* cost showed up twice (two rebuilds, the second one larger because the brain grew) and nothing is set aside.
* Otherwise the slice with the largest excess is lowered to the second-largest.
* - The lowered tenth never drops below the Theil-Sen line through the ten tenth means, so a cost that lands once in
* every tenth and grows with the brain is kept in full: those tenth means rise on that line.
*
* Nothing else changes: every other slice counts, and with no stall the measure is the plain mean of each tenth.
*/
export function excusedStall(perPageMs: number[]): ExcusedStall | undefined {
const n = perPageMs.length;
if (n < 100) return undefined;
const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
const median = (xs: number[]) => {
const s = [...xs].sort((a, b) => a - b);
const mid = s.length >> 1;
return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2;
};
const tenParts = (from: number, to: number) => Array.from({ length: 10 }, (_, k) => {
const lo = from + Math.floor((to - from) * k / 10);
const hi = from + Math.floor((to - from) * (k + 1) / 10);
return { lo, hi, mean: sum(perPageMs.slice(lo, hi)) / (hi - lo) };
});
const tenths = tenParts(0, n);
const slices = tenths.flatMap((tenth, d) => {
const parts = tenParts(tenth.lo, tenth.hi);
const typical = median(parts.map(s => s.mean));
return parts.map(s => ({ ...s, d, excess: s.mean - typical }));
}).sort((a, b) => b.excess - a.excess);
const [worst, next] = [slices[0]!, slices[1]!];
if (next.excess > worst.excess / 4) return undefined;
const means = tenths.map(t => t.mean);
const slopes = means.flatMap((a, i) => means.slice(i + 1).map((b, k) => (b - a) / (k + 1)));
const slope = median(slopes);
const trend = median(means.map((m, d) => m - slope * d)) + slope * worst.d;
const tenth = tenths[worst.d]!;
const pages = tenth.hi - tenth.lo;
const lowered = tenth.mean - (worst.excess - next.excess) * (worst.hi - worst.lo) / pages;
const ms = Math.round((tenth.mean - Math.min(tenth.mean, Math.max(lowered, trend))) * pages);
return ms > 0 ? { ms, first_page: worst.lo + 1, last_page: worst.hi } : undefined;
}
export function rateMeasure(perPageMs: number[]): RateMeasure {
const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
const excused = excusedStall(perPageMs);
const judged = [...perPageMs];
if (excused) {
const stalled = judged.slice(excused.first_page - 1, excused.last_page);
const keep = 1 - excused.ms / sum(stalled);
stalled.forEach((ms, k) => { judged[excused.first_page - 1 + k] = ms * keep; });
}
const tenth = Math.max(1, Math.floor(judged.length / 10));
const first = sum(judged.slice(0, tenth)) / tenth;
const last = sum(judged.slice(-tenth)) / tenth;
const halfPages = Math.floor(perPageMs.length / 2);
const round = (n: number, places: number) => Math.round(n * 10 ** places) / 10 ** places;
return { per_page_ms_first10: round(first, 1), per_page_ms_last10: round(last, 1), rate_ratio: round(last / first, 2),
total_ms: Math.round(sum(perPageMs)), ms_at_half: Math.round(sum(perPageMs.slice(0, halfPages))),
total_vs_half: round(sum(judged) / sum(judged.slice(0, halfPages)), 2),
...(excused ? { excused } : {}) };
}
/**
* The timing basis the import-rate gate judges. PGLite runs on the import process's main thread, so
* that thread's CPU time holds all the import's work and none of the time a shared host takes the CPU away, which
* inflates wall time on CI runners. Postgres does its work in the server, outside that process's
* CPU time, so it keeps wall time.
*/
export function importRateBasis(engine: ScaleReport['engine']): 'cpu' | 'wall' {
return engine === 'pglite' ? 'cpu' : 'wall';
}
export const LOOPS_PER_PAGE_MAX = 10;
export const HOT_TABLES = ['pages', 'links', 'content_chunks', 'timeline_entries', 'facts', 'takes'] as const;
/**
* The planner-stats gate checks only hot tables holding more than this many rows: F4b analyzes a
* table once its pending row count passes max(500, 10% of its rows), so a smaller table may
* correctly have no statistics yet.
*/
export const PLANNER_STATS_MIN_ROWS = PLANNER_STATS_MIN_PENDING;
/** The plans the planner-health gate inspects (every captured read statement of these ops). */
export const KEY_PLAN_OPS = ['get_backlinks', 'search (MCP path, remote)', 'get_health', 'find_orphans'] as const;
export const HEADLINE_OP = 'search (MCP path, remote)';
/**
* Interactive ceilings (p50 ms), defined up to 20k pages; report-only until trend.ts says stable. 20k joined
* once the Postgres MCP search slowdown between 10k and 20k stopped reproducing (nightly 2026-10-15, run
* 38708604843, at 20k: MCP search 36.3 ms Postgres / 71.1 ms PGLite, get_health 246 / 585 ms).
*/
export const CEILINGS_MS: Record<string, number> = { 'search (MCP path, remote)': 500, get_health: 1000 };
export const CEILING_PAGES = 20_000;
/** The interactive ceilings that apply at a brain size: they bind brains up to 20k pages (both nightly tiers). */
export function ceilingsFor(pages: number): Record<string, number> {
return pages <= CEILING_PAGES ? CEILINGS_MS : {};
}
/** Calibrated budgets are this multiple of the first calibration run's p50 (O-CEO-9). */
export const BUDGET_MULTIPLIER = 3;
/** Phase ceilings: 10k pages <= 20 min per engine, 50k <= 100 min (import 75, budgets 25); linear in pages, floor 2 min. */
export function phaseLimitsMs(pages: number): Record<'import' | 'budgets', number> {
const minutes = (perThousand: number) => Math.max(2, (pages / 1000) * perThousand) * 60_000;
return { import: minutes(1.5), budgets: minutes(0.5) };
}
export interface PlanStatement { sql: string; execution_ms: number; inner_loops: number; text?: string }
export interface OpPlan { statements: number; slowest?: PlanStatement; worst_loops?: PlanStatement }
export interface OpResult {
op: string;
p50_ms: number;
runs_ms: number[];
known_answer: 'pass' | 'fail';
detail?: string;
plan?: OpPlan;
}
export interface DataCheck { check: string; status: 'pass' | 'fail'; detail?: string }
export interface ScaleReport {
engine: 'pglite' | 'postgres';
pages: number;
seed: number;
import_mode: 'cli' | 'content';
/** Both timing bases, reported for every engine; `importRateBasis` picks the one the gate judges. */
import: { wall: RateMeasure; cpu: RateMeasure };
/**
* Read after the first timed op (`probed_after`), not right after import: F4b analyzes on the
* first planner-sensitive read by design. `hot_table_rows` is each table's row count.
*/
planner: { hot_table_stat_rows: Record<string, number>; hot_table_rows: Record<string, number>; probed_after: string };
ops: OpResult[];
data: DataCheck[];
/** F4d operational-ceiling measurements by check (scripts/scale/f4d.ts). */
f4d?: Record<string, Record<string, unknown>>;
phases_ms: Record<string, number>;
budgets_ms?: Record<string, number>;
}
export interface GatePolicy { enforce: boolean; enforcePlanner: boolean; enforceCeilings: boolean }
export interface GateResult {
gate: string;
status: 'pass' | 'fail';
/** Whether a failure of this gate fails the run (only meaningful under --enforce). */
enforced: boolean;
message: string;
explain?: string;
}
export interface GateVerdict { results: GateResult[]; failures: GateResult[]; reportOnlyBreaches: GateResult[]; exitCode: 0 | 1 }
export function reproduceCommand(report: Pick<ScaleReport, 'engine' | 'pages' | 'seed' | 'import_mode'>): string {
return `bun run test:scale -- --engine ${report.engine} --pages ${report.pages} --seed ${report.seed}`
+ `${report.import_mode === 'content' ? ' --import-mode content' : ''} --enforce`;
}
function planText(plan: PlanStatement | undefined): string | undefined {
if (!plan) return undefined;
return `-- ${plan.sql.replace(/\s+/g, ' ').trim()}\n${plan.text ?? '(no EXPLAIN text captured)'}`;
}
export function evaluateScaleGates(report: ScaleReport, policy: GatePolicy): GateVerdict {
const repro = reproduceCommand(report);
const results: GateResult[] = [];
const add = (gate: string, ok: boolean, enforced: boolean, message: string, explain?: string) =>
results.push({ gate, status: ok ? 'pass' : 'fail', enforced, message, ...(explain && !ok ? { explain } : {}) });
const basis = importRateBasis(report.engine);
const other = basis === 'cpu' ? 'wall' : 'cpu';
const { rate_ratio, total_vs_half, per_page_ms_first10, per_page_ms_last10, excused } = report.import[basis];
add('import_rate', rate_ratio <= RATE_RATIO_MAX && total_vs_half <= TOTAL_VS_HALF_MAX, report.pages >= RATE_MIN_PAGES,
`import rate (${basis} time): last 10% per-page cost ${per_page_ms_last10} ms is ${rate_ratio}x the first 10% (${per_page_ms_first10} ms; gate <= ${RATE_RATIO_MAX}); `
+ `total import time is ${total_vs_half}x the time at the halfway mark (gate <= ${TOTAL_VS_HALF_MAX}). `
+ (excused ? `Both ratios leave out one ${excused.ms} ms stall in pages ${excused.first_page}-${excused.last_page} that the rest of the import does not repeat (#7939). ` : '')
+ `Not judged on ${report.engine}: ${other} time ratio ${report.import[other].rate_ratio}, total/half ${report.import[other].total_vs_half}. `
+ `A rising per-page cost means import slows as the brain grows, usually stale planner statistics during import. Reproduce: ${repro}`);
const { hot_table_stat_rows: statRows, hot_table_rows: tableRows, probed_after: probedAfter } = report.planner;
const missing = HOT_TABLES.filter(t => tableRows[t]! > PLANNER_STATS_MIN_ROWS && !(statRows[t]! > 0));
// Postgres statistics belong to autovacuum, whose timing gbrain does not control (F4b is PGLite-only):
// missing pg_stats rows there are reported, never a failure.
add('planner_stats', missing.length === 0, policy.enforcePlanner && report.engine === 'pglite',
missing.length === 0 ? `planner stats: every hot table above ${PLANNER_STATS_MIN_ROWS} rows has pg_stats rows after ${probedAfter}`
: `planner stats: no pg_stats rows after ${probedAfter} for ${missing.map(t => `${t} (${tableRows[t]} rows)`).join(', ')}; `
+ 'the planner is guessing row counts on these tables. '
+ (report.engine === 'pglite'
? 'Import and the first planner-sensitive read must leave statistics behind (F4b); '
+ `check \`gbrain doctor\` planner_stats_stale and run \`gbrain repair planner-stats --apply\` to confirm. Reproduce: ${repro}`
: `Postgres autovacuum collects them on its own schedule, so this is report-only there. Reproduce: ${repro}`));
const loopsLimit = LOOPS_PER_PAGE_MAX * report.pages;
for (const name of KEY_PLAN_OPS) {
const op = report.ops.find(o => o.op === name);
const worst = op?.plan?.worst_loops;
const loops = worst?.inner_loops ?? 0;
add(`planner_loops:${name}`, loops <= loopsLimit, policy.enforcePlanner,
`planner loops: ${name} key plan worst Nested Loop inner loops ${loops} (gate <= ${loopsLimit}, ${LOOPS_PER_PAGE_MAX}x pages). `
+ `A Nested Loop rescanning its inner side per row scales quadratically. Reproduce: ${repro}`, planText(worst));
}
for (const op of report.ops) {
add(`known_answer:${op.op}`, op.known_answer === 'pass', true,
op.known_answer === 'pass' ? `known answer: ${op.op} returned the expected result`
: `known answer: ${op.op} returned a wrong answer: ${op.detail ?? 'no detail'}. Reproduce: ${repro}`,
planText(op.plan?.slowest));
}
for (const check of report.data) {
add(`data:${check.check}`, check.status === 'pass', true,
check.status === 'pass' ? `data check: ${check.check} holds` : `data check: ${check.check} failed: ${check.detail ?? 'no detail'}. Reproduce: ${repro}`);
}
const limits = phaseLimitsMs(report.pages);
for (const phase of ['import', 'budgets'] as const) {
const ms = report.phases_ms[phase] ?? 0;
// The budgets phase times the ops, so un-analyzed plans dominate it: it is stats-dependent and flips with planner health.
add(`phase:${phase}`, ms <= limits[phase], phase === 'import' || policy.enforcePlanner,
`phase timer: ${phase} took ${Math.round(ms / 1000)} s (ceiling ${Math.round(limits[phase] / 1000)} s at ${report.pages} pages). Reproduce: ${repro}`);
}
for (const [name, ceiling] of Object.entries(ceilingsFor(report.pages))) {
const op = report.ops.find(o => o.op === name);
if (!op) continue;
add(`ceiling:${name}`, op.p50_ms < ceiling, policy.enforceCeilings,
`interactive ceiling: ${name} p50 ${op.p50_ms} ms (ceiling < ${ceiling} ms). Reproduce: ${repro}`, planText(op.plan?.slowest));
}
for (const [name, budget] of Object.entries(report.budgets_ms ?? {})) {
const op = report.ops.find(o => o.op === name);
if (!op) continue;
add(`budget:${name}`, op.p50_ms <= budget, policy.enforceCeilings,
`calibrated budget: ${name} p50 ${op.p50_ms} ms (budget ${budget} ms, ${BUDGET_MULTIPLIER}x the calibration run). Reproduce: ${repro}`,
planText(op.plan?.slowest));
}
const failures = results.filter(r => r.status === 'fail' && r.enforced);
const reportOnlyBreaches = results.filter(r => r.status === 'fail' && !r.enforced);
return { results, failures, reportOnlyBreaches, exitCode: policy.enforce && failures.length > 0 ? 1 : 0 };
}
/** The lines a run prints for its verdict: every failure with its EXPLAIN, then report-only breaches, then the outcome. */
export function verdictLines(report: ScaleReport, verdict: GateVerdict, policy: GatePolicy): string[] {
const lines: string[] = [];
for (const f of verdict.failures) {
lines.push(`[scale] GATE FAIL ${f.gate}: ${f.message}`);
if (f.explain) lines.push(...f.explain.split('\n').map(l => `[scale] ${l}`));
}
for (const b of verdict.reportOnlyBreaches) lines.push(`[scale] REPORT-ONLY ${b.gate}: ${b.message}`);
if (!policy.enforce) {
lines.push(`[scale] report-only run: exit 0 regardless of gates (${verdict.failures.length} gate(s) would fail under --enforce).`);
} else if (verdict.failures.length > 0) {
lines.push(`[scale] ${verdict.failures.length} enforced gate(s) failed; exit 1. Fix the named op or phase, then rerun: ${reproduceCommand(report)}`);
} else {
lines.push(`[scale] all enforced gates passed (planner health and the budgets phase timer ${policy.enforcePlanner ? 'enforced' : 'report-only (PLANNER_HEALTH_ENFORCED=false)'}, `
+ `ceilings ${policy.enforceCeilings ? 'enforced' : 'report-only until GBRAIN_SCALE_ENFORCE_CEILINGS=1'}).`);
}
return lines;
}
/** Every `{slug, source_id?}` an op result carries, in order (op results nest hits in arrays or evidence objects). */
export function resultHits(value: unknown): Array<{ slug: string; source_id?: string }> {
const hits: Array<{ slug: string; source_id?: string }> = [];
const walk = (v: unknown) => {
if (Array.isArray(v)) { for (const item of v) walk(item); return; }
if (!v || typeof v !== 'object') return;
const o = v as Record<string, unknown>;
if (typeof o.slug === 'string') hits.push({ slug: o.slug, ...(typeof o.source_id === 'string' ? { source_id: o.source_id } : {}) });
for (const child of Object.values(o)) if (child && typeof child === 'object') walk(child);
};
walk(value);
return hits;
}
/**
* The find_orphans known-answer call: one page at the op's maximum size. The
* op returns a page of rows (default 100) plus `total_orphans`, and recomputes
* the whole orphan set per call, so the check reads totals and one maximal page
* instead of paging (a 20k brain holds more orphans than one default page).
*/
export const FIND_ORPHANS_PARAMS = { limit: ORPHANS_MAX_LIMIT } as const;
/** Null when every fixture island is among the orphans, else what is wrong. */
export function orphansProblem(result: unknown, islands: readonly string[]): string | null {
const r = (result ?? {}) as { orphans?: Array<{ slug?: string }>; total_orphans?: unknown };
const rows = Array.isArray(r.orphans) ? r.orphans : [];
const total = Number(r.total_orphans);
if (!Number.isInteger(total)) return 'find_orphans returned no total_orphans';
if (total < islands.length) return `find_orphans counts ${total} orphans, fewer than the fixture's ${islands.length} island pages`;
if (rows.length < total) {
return `find_orphans returned ${rows.length} of ${total} orphans in one call; the known-answer check reads every orphan in one page `
+ `of at most ${ORPHANS_MAX_LIMIT}, so this fixture size needs a paged check`;
}
const found = new Set(rows.map(h => h.slug));
const missing = islands.filter(slug => !found.has(slug));
return missing.length ? `${missing.length} of ${islands.length} island pages missing from find_orphans (first: ${missing[0]})` : null;
}