Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Agent contract v1 D5: the CLI contract over src/cli/command-table.ts.
*
* Part 1 (help): `gbrain <command> --help` for every CLI-only command, with
* no brain configured, must exit 0, print real (non-stub) help, list only
* curated flags when the command has a curated help module, and mention
* `--yes` when the command parses it. Commands that do not meet the contract
* yet are listed in a shrink-only baseline
* (test/fixtures/cli-contract/help-baseline.json): a NEW violation fails, and
* a baseline entry that now passes must be removed
* (`GBRAIN_TEST_UPDATE_GOLDENS=1 bun test test/cli-contract.test.ts`
* rewrites it, refusing growth).
*
* Part 2 (json): every record that declares `json` has one succeeding and one
* failing invocation here; the success prints the declared shape (one JSON
* document, or NDJSON lines) and the failure prints a v1 envelope with `code`
* and `suggestion` (or, exit 3, the confirmation_required payload).
* Consent-gated exits (3) per command are pinned by
* test/consent-table.serial.test.ts (C9).
*
* Serial: spawns the CLI and writes temporary GBRAIN_HOMEs.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { CLI_COMMANDS } from '../src/cli/command-table.ts';
import { CLI_FLAG_REGISTRY } from '../src/core/cli-flag-registry.generated.ts';
import { runCli, runCliBatch, type CliResult } from './helpers/cli-spawn.ts';
const BASELINE = join(import.meta.dir, 'fixtures', 'cli-contract', 'help-baseline.json');
const STUB = 'run gbrain --help for the full command list';
const emptyHome = mkdtempSync(join(tmpdir(), 'gbrain-cli-contract-empty-'));
const brainHome = mkdtempSync(join(tmpdir(), 'gbrain-cli-contract-brain-'));
afterAll(() => {
rmSync(emptyHome, { recursive: true, force: true });
rmSync(brainHome, { recursive: true, force: true });
});
/**
* The command parses `--yes`: the generated acceptance registry lists it AND
* the command's own module reads the literal (the registry over-approximates
* from every module a command imports).
*/
function parsesYes(name: string): boolean {
if (!(CLI_FLAG_REGISTRY[name] ?? []).includes('--yes')) return false;
const src = join(import.meta.dir, '..', 'src');
const files = [join(src, 'cli', 'commands', `${name}.ts`), join(src, 'commands', `${name}.ts`), join(src, 'commands', name, 'index.ts')];
return files.some(f => existsSync(f) && /['"]--yes['"]/.test(readFileSync(f, 'utf8')));
}
/** The contract violations of one command's --help output. */
async function helpViolations(name: string, r: CliResult): Promise<string[]> {
const out = `${r.stdout}\n${r.stderr}`;
const v: string[] = [];
if (r.exitCode !== 0) v.push('exit');
if (out.includes(STUB) || out.split('\n').filter(l => l.trim()).length < 3) v.push('stub');
const record = CLI_COMMANDS.find(c => c.name === name)!;
if (record.help) {
const curated = new Set(['--help', ...(await record.help()).help.flags.map(f => f.name)]);
const listed = [...out.matchAll(/(?<![\w-])--[a-z][a-z0-9-]*/g)].map(m => m[0]);
if (listed.some(f => !curated.has(f))) v.push('uncurated_flag');
}
if (parsesYes(name) && !out.includes('--yes')) v.push('missing_yes');
return v.map(x => `${name}:${x}`);
}
describe('D5 help contract (shrink-only baseline)', () => {
test('every CLI-only command: --help exits 0, is not a stub, lists curated flags, shows --yes where parsed', async () => {
const names = CLI_COMMANDS.map(c => c.name);
const results = await runCliBatch(names.map(n => [n, '--help']), { home: emptyHome, cwd: emptyHome, width: 3, timeoutMs: 60_000 });
const violations = (await Promise.all(names.map((n, i) => helpViolations(n, results[i])))).flat().sort();
const baseline: string[] = existsSync(BASELINE) ? JSON.parse(readFileSync(BASELINE, 'utf8')) : [];
const grown = violations.filter(v => !baseline.includes(v));
if (process.env.GBRAIN_TEST_UPDATE_GOLDENS === '1' && (grown.length === 0 || !existsSync(BASELINE))) {
mkdirSync(join(import.meta.dir, 'fixtures', 'cli-contract'), { recursive: true });
writeFileSync(BASELINE, `${JSON.stringify(violations, null, 2)}\n`);
return;
}
expect(grown, 'new help-contract violations (fix the help; the baseline only shrinks)').toEqual([]);
expect(baseline.filter(b => !violations.includes(b)), 'fixed: remove these from test/fixtures/cli-contract/help-baseline.json').toEqual([]);
}, 600_000);
});
interface JsonRow {
ok: string[];
fail: string[];
/** The failing invocation runs against the keyless brain (default: an empty home). */
failOnBrain?: boolean;
/** The succeeding invocation runs against an empty home (default: the keyless brain). */
okOnEmpty?: boolean;
/** The succeeding invocation needs Postgres: the E2E file (test/e2e/…) that asserts it instead. */
okE2E?: string;
/** NDJSON only: zero lines is a valid success (e.g. an export of an empty table). */
okMayBeEmpty?: true;
}
const EVAL_FIXTURES = join(import.meta.dir, 'fixtures', 'eval-baselines');
const QRELS = join(EVAL_FIXTURES, 'qrels-search.json');
const CAPTURED = join(EVAL_FIXTURES, 'captured-sample.ndjson');
/** One row per command-table record that declares `json`; a new declaration needs a row. */
const JSON_ROWS: Record<string, JsonRow> = {
errors: { ok: ['errors', 'unknown_flag', '--json'], fail: ['errors', 'definitely_not_a_code', '--json'], okOnEmpty: true },
init: { ok: ['init', '--pglite', '--no-embedding', '--json'], fail: ['init', '--mcp-only', '--json', '--mcp-url', 'http://127.0.0.1:1/mcp'], okOnEmpty: true },
doctor: { ok: ['doctor', '--json', '--fast'], fail: ['doctor', '--json'] },
sync: { ok: ['sync', '--source', 'notes', '--no-pull', '--json'], fail: ['sync', '--source', 'no-such-source', '--no-pull', '--json'], failOnBrain: true },
embed: { ok: ['embed', '--stale', '--json'], fail: ['embed', '--all', '--json'], failOnBrain: true },
'db-repair': { ok: ['db-repair', '--json'], fail: ['db-repair', '--json'], failOnBrain: true, okE2E: 'test/e2e/cli-json-commands-postgres.test.ts' },
dream: { ok: ['dream', '--json', '--phase', 'lint'], fail: ['dream', '--json', '--phase', 'garbage'] },
'post-upgrade': { ok: ['post-upgrade', '--json', '--no-autopilot-install'], fail: ['post-upgrade', '--bogus', '--json'], okOnEmpty: true },
bench: { ok: ['bench', 'publish', '--from', CAPTURED, '--to', 'sample.baseline.ndjson', '--json'], fail: ['bench', 'publish', '--json'], okOnEmpty: true },
'eval export': { ok: ['eval', 'export', '--json'], fail: ['eval', 'export', '--json', '--since', 'bogus'], failOnBrain: true, okMayBeEmpty: true },
'eval replay': { ok: ['eval', 'replay', '--against', CAPTURED, '--json'], fail: ['eval', 'replay', '--json'], failOnBrain: true },
'eval gate': {
ok: ['eval', 'gate', '--qrels', QRELS, '--embedder', 'deterministic', '--threshold-recall-at-k', '0', '--threshold-first-relevant-hit', '0', '--threshold-expected-top1', '0', '--json'],
fail: ['eval', 'gate', '--qrels', QRELS, '--embedder', 'deterministic', '--json'], failOnBrain: true,
},
// Lane H journey: import declares json (one document; a keyless brain's refusal is an embedding_disabled envelope).
import: { ok: ['import', join(brainHome, 'loose'), '--no-embed', '--json'], fail: ['import', join(brainHome, 'loose'), '--json'], failOnBrain: true },
'apply-migrations': { ok: ['apply-migrations', '--dry-run', '--json'], fail: ['apply-migrations', '--json', '--migration', '9.9.9'], failOnBrain: true },
// #5575: the backfill dry run is read-only; explaining a ref that names nothing is a not_found envelope.
trust: { ok: ['trust', 'backfill', '--dry-run', '--json'], fail: ['trust', 'explain', 'f999999', '--json'], failOnBrain: true },
// W1 belief ledger: status is read-only; an unknown source is an invalid_params envelope.
beliefs: { ok: ['beliefs', 'status', '--source', 'notes', '--json'], fail: ['beliefs', 'status', '--source', 'no-such-source', '--json'], failOnBrain: true },
// GBRA-151: the bundle is read-only and bounded; a budget that is not a positive number is an invalid_params envelope.
diag: { ok: ['diag', '--json', '--budget-ms', '20000'], fail: ['diag', '--json', '--budget-ms', '0'], failOnBrain: true },
// D8: the dry run resolves and prints the target without writing; an unwired harness id is a coded refusal.
setup: { ok: ['setup', 'claude-code', '--dry-run', '--json', '--gbrain-bin', process.execPath], fail: ['setup', 'codex', '--json'], okOnEmpty: true },
// Spring-loaded skills: the catalog listing reads the binary only; an unknown skill name is a skill_not_found envelope.
'bundled-skill': { ok: ['bundled-skill', '--json'], fail: ['bundled-skill', 'definitely-not-a-skill', '--json'], okOnEmpty: true },
};
function parsedShape(mode: 'document' | 'ndjson', stdout: string): unknown[] {
if (mode === 'document') return [JSON.parse(stdout)];
return stdout.split('\n').filter(l => l.trim() !== '').map(l => JSON.parse(l));
}
/** json-declared records, plus `<command> <sub>` for each per-subcommand declaration. */
const JSON_DECLARED: ReadonlyArray<{ name: string; mode: 'document' | 'ndjson' }> = CLI_COMMANDS.flatMap(c => c.json
? [{ name: c.name, mode: c.json }]
: Object.entries(c.jsonSubcommands ?? {}).map(([sub, mode]) => ({ name: `${c.name} ${sub}`, mode })));
function assertFailureShape(name: string, mode: 'document' | 'ndjson', r: CliResult): void {
expect(r.exitCode, `${name}: failing invocation exits non-zero`).not.toBe(0);
const docs = parsedShape(mode, r.stdout) as Array<Record<string, unknown>>;
const last = docs[docs.length - 1];
if (r.exitCode === 3) {
expect(last, `${name}: exit 3 carries the consent payload`).toMatchObject({ code: 'confirmation_required' });
return;
}
expect(typeof last.code, `${name}: failure document has code`).toBe('string');
expect(typeof last.suggestion, `${name}: failure document has suggestion`).toBe('string');
}
describe('D5 json contract: one success and one failure per json-declared command', () => {
beforeAll(async () => {
const init = await runCli(['init', '--pglite', '--no-embedding', '--json'], { home: brainHome, cwd: brainHome, timeoutMs: 120_000 });
if (init.exitCode !== 0) throw new Error(`fixture init failed: ${init.stderr}`);
const repo = join(brainHome, 'notes');
mkdirSync(repo);
writeFileSync(join(repo, 'example.md'), '---\ntitle: Example\n---\nHello world\n');
const git = (args: string[]) => Bun.spawnSync(['git', '-c', '[email protected]', '-c', 'user.name=fixture', ...args], { cwd: repo });
git(['init', '-q']); git(['add', '.']); git(['commit', '-qm', 'init']);
mkdirSync(join(brainHome, 'loose'));
writeFileSync(join(brainHome, 'loose', 'loose-page.md'), '---\ntitle: Loose\n---\nA page outside any source repo\n');
const add = await runCli(['sources', 'add', 'notes', '--path', repo], { home: brainHome, cwd: brainHome, timeoutMs: 60_000 });
if (add.exitCode !== 0) throw new Error(`fixture sources add failed: ${add.stderr}`);
}, 240_000);
test('every json-declared record has a contract row', () => {
const declared = JSON_DECLARED.map(c => c.name).sort();
expect(declared).toEqual(Object.keys(JSON_ROWS).sort());
});
for (const record of JSON_DECLARED) {
test(`${record.name} --json: success shape and failure envelope`, async () => {
const row = JSON_ROWS[record.name];
if (!row) throw new Error(`no D5 row for json-declared command ${record.name}`);
const fresh = mkdtempSync(join(tmpdir(), 'gbrain-cli-contract-ok-'));
try {
const okHome = row.okOnEmpty ? fresh : brainHome;
if (row.okE2E) {
expect(existsSync(join(import.meta.dir, '..', row.okE2E)), `${record.name}: ${row.okE2E} covers the success shape`).toBe(true);
} else {
const ok = await runCli(row.ok, { home: okHome, cwd: okHome, timeoutMs: 120_000 });
expect(ok.exitCode, `${record.name} ok: ${ok.stderr.slice(-800)}`).toBe(0);
const lines = parsedShape(record.mode, ok.stdout).length;
if (!row.okMayBeEmpty) expect(lines).toBeGreaterThan(0);
}
const failHome = row.failOnBrain ? brainHome : emptyHome;
assertFailureShape(record.name, record.mode, await runCli(row.fail, { home: failHome, cwd: failHome, timeoutMs: 120_000 }));
} finally {
rmSync(fresh, { recursive: true, force: true });
}
}, 300_000);
}
});