Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* H2 (test-gap wave) — runPhaseSkillopt's flag gate + cost caps: the only
* brake on an unattended nightly LLM spender (`gbrain dream --phase skillopt`).
*
* Pins, exactly as coded in src/core/skillopt/cycle-phase.ts:
* 1. `cycle.skillopt.enabled` unset / non-'true' → status 'skipped',
* details.reason 'feature_flag_off', ZERO runner invocations.
* 2. `once: true` bypasses the flag for that call only — the flag key is
* never written (per-skill `last_run` state IS still banked; that is
* re-entry bookkeeping, not the feature flag).
* 3. Brain-wide cap boundary is `cumulativeCostUsd >= brainWideCap`,
* checked BEFORE each candidate runs: two $1.00 spends against a $2.00
* cap put cumulative exactly AT the cap, so the third candidate is
* skipped with reason 'brain_wide_cap_reached' (a strict `>` would have
* run it). Strictly below the cap ($1.99) the third still runs.
* 4. Effective per-skill cap = Math.min(perSkillCap, brainWideCap −
* cumulative) — the runner receives the min, not the raw per-skill cap.
* 5. An aborted signal breaks the loop between candidates: no further
* runner calls, no synthetic 'skipped' rows for the unvisited tail.
*
* Serial lane (isolation rule R2): cycle-phase.ts imports runSkillOpt
* STATICALLY from ./orchestrator.ts (no injected-runner seam), so mock.module
* is the only stub. The stub is mandatory — an unmocked run would launch a
* real LLM optimizer. autoDetectSkillsDirReadOnly is mocked to a temp
* fixture tree so the phase never walks the repo's real skills/.
* cycle-phase.ts is loaded via dynamic import AFTER mock registration so the
* static bindings resolve to the mocks.
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, mock } from 'bun:test';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { errorFor } from '../../src/core/errors.ts';
import { resetPgliteState } from '../helpers/reset-pglite.ts';
// ─── Fixture skills trees (benchmark file existence is all the phase reads;
// the stubbed runner never opens them) ─────────────────────────────────
function makeSkillsDir(prefix: string, skillNames: string[]): string {
const root = mkdtempSync(join(tmpdir(), prefix));
for (const name of skillNames) {
mkdirSync(join(root, name));
writeFileSync(join(root, name, 'skillopt-benchmark.jsonl'), '{}\n');
}
return root;
}
const THREE_SKILLS = ['skill-a', 'skill-b', 'skill-c'];
const skills3Dir = makeSkillsDir('cycle-phase-caps-3-', THREE_SKILLS);
const skills2Dir = makeSkillsDir('cycle-phase-caps-2-', ['skill-a', 'skill-b']);
const skills1Dir = makeSkillsDir('cycle-phase-caps-1-', ['skill-a']);
// ─── Mocks (registered before cycle-phase.ts is loaded) ────────────────────
let currentSkillsDir: string = skills3Dir;
let runnerCalls: Array<Record<string, unknown>> = [];
/** Per-call reported spend, indexed by call order; missing → $0. */
let stubCosts: number[] = [];
/** Optional per-call hook (used by the abort tests). */
let onRunnerCall: ((opts: Record<string, unknown>, idx: number) => void) | null = null;
/** Optional per-call result override (used by the errored-run tests). */
let stubResult: ((idx: number) => { outcome: string; receipt: Record<string, unknown> } | null) | null = null;
mock.module('../../src/core/skillopt/orchestrator.ts', () => ({
runSkillOpt: async (opts: Record<string, unknown>) => {
const idx = runnerCalls.length;
runnerCalls.push(opts);
onRunnerCall?.(opts, idx);
const override = stubResult?.(idx);
if (override) return { ...override, finalText: 'stub', mutatedSkillFile: false };
return {
outcome: 'accepted',
receipt: { final_cost_usd: stubCosts[idx] ?? 0 },
finalText: 'stub',
mutatedSkillFile: false,
};
},
}));
mock.module('../../src/core/repo-root.ts', () => ({
autoDetectSkillsDirReadOnly: () => ({ dir: currentSkillsDir, source: 'cwd_walk' }),
}));
const { runPhaseSkillopt } = await import('../../src/core/skillopt/cycle-phase.ts');
interface PhaseDetails {
reason?: string;
skills_scanned?: number;
skipped_brain_wide_cap?: number;
cumulative_cost_usd?: number;
brain_wide_cap_usd?: number;
per_skill_cap_usd?: number;
results?: Array<{ skill: string; outcome: string; cost_usd: number; reason?: string }>;
}
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
if (engine) await engine.disconnect();
for (const dir of [skills3Dir, skills2Dir, skills1Dir]) {
rmSync(dir, { recursive: true, force: true });
}
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
currentSkillsDir = skills3Dir;
runnerCalls = [];
stubCosts = [];
onRunnerCall = null;
stubResult = null;
});
async function enableFlag(): Promise<void> {
await engine.setConfig('cycle.skillopt.enabled', 'true');
}
describe('runPhaseSkillopt feature-flag gate', () => {
test('flag unset → skipped/feature_flag_off with zero runner invocations', async () => {
const res = await runPhaseSkillopt({ engine });
expect(res.phase).toBe('skillopt');
expect(res.status).toBe('skipped');
// Pin the real user-facing strings — the skip must tell the operator the
// exact enable command.
expect(res.summary).toBe(
'feature flag off (gbrain config set cycle.skillopt.enabled true to enable)',
);
expect(res.details).toEqual({ reason: 'feature_flag_off' });
expect(runnerCalls.length).toBe(0);
});
test("only the strict string 'true' enables — 'false', '1', 'TRUE' all stay off", async () => {
for (const value of ['false', '1', 'TRUE']) {
await engine.setConfig('cycle.skillopt.enabled', value);
const res = await runPhaseSkillopt({ engine });
expect(res.status).toBe('skipped');
expect((res.details as PhaseDetails).reason).toBe('feature_flag_off');
}
expect(runnerCalls.length).toBe(0);
});
test('once: true runs with the flag unset, WITHOUT writing the flag key', async () => {
currentSkillsDir = skills1Dir;
stubCosts = [0.05];
const res = await runPhaseSkillopt({ engine, once: true });
expect(res.status).toBe('ok');
expect(runnerCalls.length).toBe(1);
// The bypass never persists: the stored flag is still unset afterwards,
// so the NEXT plain cycle run skips again.
expect(await engine.getConfig('cycle.skillopt.enabled')).toBeNull();
const followUp = await runPhaseSkillopt({ engine });
expect(followUp.status).toBe('skipped');
expect(runnerCalls.length).toBe(1);
// Re-entry bookkeeping (NOT the feature flag) is still banked: last_run
// is written so nightly cycles stay cheap to re-enter.
const lastRun = await engine.getConfig('cycle.skillopt.last_run.skill-a');
expect(lastRun).not.toBeNull();
expect(Number(lastRun)).toBeGreaterThan(0);
});
});
describe('runPhaseSkillopt cost caps', () => {
test('anti-vacuity control: ample caps run all three candidates with the default per-skill cap', async () => {
await enableFlag();
stubCosts = [0.10, 0.10, 0.10];
const res = await runPhaseSkillopt({ engine });
expect(res.status).toBe('ok');
expect(runnerCalls.length).toBe(3);
// Default caps: per-skill $0.50, brain-wide $2.00. Remaining never dips
// below the per-skill cap here, so every call receives the full $0.50.
for (const call of runnerCalls) {
expect(call.maxCostUsd).toBe(0.50);
}
// Nightly-safety shape the runner is invoked with: one epoch, no-mutate.
expect(runnerCalls[0]!.epochs).toBe(1);
expect(runnerCalls[0]!.noMutate).toBe(true);
expect(runnerCalls[0]!.allowMutateBundled).toBe(false);
const d = res.details as PhaseDetails;
expect(d.skipped_brain_wide_cap).toBe(0);
expect(d.per_skill_cap_usd).toBe(0.50);
expect(d.brain_wide_cap_usd).toBe(2.00);
expect(d.cumulative_cost_usd).toBeCloseTo(0.30, 10);
expect(res.summary).toBe(
'optimized 3/3 skills (0 no-improvement, 0 errored, 0 skipped over brain-wide cap)',
);
});
test('brain-wide cap boundary is >= (at-cap skips): 3 × $1.00 against $2.00 skips the third', async () => {
await enableFlag();
await engine.setConfig('cycle.skillopt.per_skill_cap_usd', '1');
await engine.setConfig('cycle.skillopt.brain_wide_cap_usd', '2');
stubCosts = [1.00, 1.00];
const res = await runPhaseSkillopt({ engine });
// After two $1.00 spends, cumulative === cap EXACTLY. The pre-run check
// is `cumulativeCostUsd >= brainWideCap`, so the third candidate never
// reaches the runner. Under a strict `>` it would have run.
expect(runnerCalls.length).toBe(2);
const d = res.details as PhaseDetails;
expect(d.skipped_brain_wide_cap).toBe(1);
expect(d.cumulative_cost_usd).toBe(2.00);
const results = d.results!;
expect(results.length).toBe(3);
// The skipped row is the THIRD iteration (readdir order is arbitrary, so
// match by position + shape, not by name), zero-cost, with the pinned reason.
expect(results[2]).toEqual({
skill: results[2]!.skill,
outcome: 'skipped',
cost_usd: 0,
reason: 'brain_wide_cap_reached',
});
expect(results[0]!.outcome).toBe('accepted');
expect(results[1]!.outcome).toBe('accepted');
expect(results.map((r) => r.skill).sort()).toEqual(THREE_SKILLS);
// A capped skip is NOT an error: status stays ok and the summary counts it.
expect(res.status).toBe('ok');
expect(res.summary).toBe(
'optimized 2/3 skills (0 no-improvement, 0 errored, 1 skipped over brain-wide cap)',
);
});
test('strictly below the cap still runs: cumulative $1.99 < $2.00 lets the third through at min-cap', async () => {
await enableFlag();
await engine.setConfig('cycle.skillopt.per_skill_cap_usd', '1');
await engine.setConfig('cycle.skillopt.brain_wide_cap_usd', '2');
stubCosts = [1.00, 0.99, 0.005];
const res = await runPhaseSkillopt({ engine });
// 1.99 >= 2.00 is false → the third candidate RUNS (the other side of
// the >= boundary), capped to the remaining ~$0.01.
expect(runnerCalls.length).toBe(3);
expect(runnerCalls[2]!.maxCostUsd as number).toBeCloseTo(0.01, 10);
expect((res.details as PhaseDetails).skipped_brain_wide_cap).toBe(0);
});
test('effective per-skill cap = min(perSkillCap, remaining): runner receives the min', async () => {
await enableFlag();
currentSkillsDir = skills2Dir;
await engine.setConfig('cycle.skillopt.per_skill_cap_usd', '1.5');
await engine.setConfig('cycle.skillopt.brain_wide_cap_usd', '2');
stubCosts = [1.00, 0.25];
await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(2);
// Call 1: remaining $2.00 > per-skill $1.50 → min is the per-skill cap.
expect(runnerCalls[0]!.maxCostUsd).toBe(1.5);
// Call 2: remaining $1.00 < per-skill $1.50 → min is the remaining budget.
expect(runnerCalls[1]!.maxCostUsd).toBe(1.0);
});
});
describe('runPhaseSkillopt reservation refusals (#5585)', () => {
const refuse = () => {
throw errorFor({ class: 'CostCapExceeded', code: 'cost_cap_exceeded', message: 'reservation_exceeds_cap: a single optimizer call reserves $0.80' });
};
test('refusal under a brain-wide-reduced cap is brain_wide_cap_reached and writes no last_skip', async () => {
await enableFlag();
currentSkillsDir = skills2Dir;
await engine.setConfig('cycle.skillopt.per_skill_cap_usd', '0.8');
await engine.setConfig('cycle.skillopt.brain_wide_cap_usd', '1');
stubCosts = [0.5];
// Call 1 spends $0.50; call 2 gets min(0.8, 0.5) = $0.50 and is refused.
onRunnerCall = (_opts, idx) => { if (idx === 1) refuse(); };
const res = await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(2);
expect(runnerCalls[1]!.maxCostUsd).toBe(0.5);
const d = res.details as PhaseDetails;
expect(d.results![1]).toEqual({ skill: d.results![1]!.skill, outcome: 'skipped', cost_usd: 0, reason: 'brain_wide_cap_reached' });
expect(d.skipped_brain_wide_cap).toBe(1);
// Not a per-skill budget skip: tomorrow's full per-skill cap may fit it.
expect(await engine.getConfig(`cycle.skillopt.last_skip.${d.results![1]!.skill}`)).toBeNull();
});
test('an unreadable benchmark never throws the phase: the skill is re-admitted', async () => {
await enableFlag();
const root = mkdtempSync(join(tmpdir(), 'cycle-phase-caps-unreadable-'));
try {
// The benchmark path exists but is a directory, so reading it fails.
mkdirSync(join(root, 'skill-x', 'skillopt-benchmark.jsonl'), { recursive: true });
currentSkillsDir = root;
await engine.setConfig('cycle.skillopt.last_skip.skill-x', '{"stale":"fingerprint"}');
const res = await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(1);
expect((res.details as PhaseDetails).results![0]!.skill).toBe('skill-x');
} finally {
rmSync(root, { recursive: true, force: true });
}
});
test('refusal at the full per-skill cap is skipped_budget and records last_skip', async () => {
await enableFlag();
currentSkillsDir = skills1Dir;
onRunnerCall = refuse;
const res = await runPhaseSkillopt({ engine });
const row = (res.details as PhaseDetails).results![0]!;
expect(row.outcome).toBe('skipped_budget');
expect(await engine.getConfig('cycle.skillopt.last_skip.skill-a')).toContain('"benchmark_sha8"');
expect(await engine.getConfig('cycle.skillopt.last_error.skill-a')).toBeNull();
});
});
describe('runPhaseSkillopt abort signal', () => {
test('abort during the first run stops the loop — no further runner calls, no skipped rows', async () => {
await enableFlag();
const controller = new AbortController();
onRunnerCall = () => controller.abort();
stubCosts = [0.10];
const res = await runPhaseSkillopt({ engine, signal: controller.signal });
expect(runnerCalls.length).toBe(1);
const d = res.details as PhaseDetails;
// break, not skip: the unvisited tail gets no synthetic result rows and
// does not count against skipped_brain_wide_cap.
expect(d.results!.length).toBe(1);
expect(d.skipped_brain_wide_cap).toBe(0);
expect(d.skills_scanned).toBe(3);
expect(res.status).toBe('ok');
});
test('already-aborted signal → zero runner invocations', async () => {
await enableFlag();
const controller = new AbortController();
controller.abort();
const res = await runPhaseSkillopt({ engine, signal: controller.signal });
expect(runnerCalls.length).toBe(0);
expect((res.details as PhaseDetails).results!.length).toBe(0);
expect(res.status).toBe('ok');
});
});
describe('runPhaseSkillopt errored runs (#5584)', () => {
const REMEDIATION = [{ code: 'reflect_truncated', fix: 'raise the cap', docs: 'docs/guides/skillopt.md#reflect_truncated' }];
const ERRORED = {
outcome: 'errored',
receipt: {
final_cost_usd: 0.2,
run_id: 'run-err',
abort_reason: 'error',
abort_detail: 'optimizer_output_unusable: reflect_failure_truncated: 32000 output tokens, max_tokens=32000',
remediation: REMEDIATION,
},
};
test('errored run forwards diagnostics, leaves last_run unset, records last_error', async () => {
await enableFlag();
currentSkillsDir = skills1Dir;
stubResult = () => ERRORED;
const res = await runPhaseSkillopt({ engine });
const row = (res.details as PhaseDetails).results![0] as Record<string, unknown>;
expect(row).toEqual({
skill: 'skill-a',
outcome: 'errored',
cost_usd: 0.2,
abort_reason: 'error',
abort_detail: ERRORED.receipt.abort_detail,
run_id: 'run-err',
remediation: REMEDIATION,
});
expect(res.status).toBe('warn');
expect(await engine.getConfig('cycle.skillopt.last_run.skill-a')).toBeNull();
expect(Number(await engine.getConfig('cycle.skillopt.last_error.skill-a'))).toBeGreaterThan(0);
});
test('an errored skill is not retried within 24h of its last error, and is retried after', async () => {
await enableFlag();
currentSkillsDir = skills1Dir;
stubResult = () => ERRORED;
await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(1);
const again = await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(1);
expect(again.summary).toBe('no stale skills with benchmarks; nothing to optimize');
await engine.setConfig('cycle.skillopt.last_error.skill-a', String(Date.now() - 25 * 3600 * 1000));
stubResult = null;
await runPhaseSkillopt({ engine });
expect(runnerCalls.length).toBe(2);
expect(await engine.getConfig('cycle.skillopt.last_run.skill-a')).not.toBeNull();
});
test('a thrown run error also records last_error instead of last_run', async () => {
await enableFlag();
currentSkillsDir = skills1Dir;
onRunnerCall = () => { throw new Error('d_sel_too_small'); };
const res = await runPhaseSkillopt({ engine });
expect((res.details as PhaseDetails).results![0]).toEqual({ skill: 'skill-a', outcome: 'errored', cost_usd: 0, reason: 'd_sel_too_small' });
expect(await engine.getConfig('cycle.skillopt.last_run.skill-a')).toBeNull();
expect(await engine.getConfig('cycle.skillopt.last_error.skill-a')).not.toBeNull();
});
});