Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* scripts/eval-spend-guard.sh — subprocess pins with a temp ledger.
*
* Hermetic: the wrapped "paid command" is a shell one-liner that writes a
* marker file (proving it ran) and optionally an actual-cost file. Env is
* passed to spawn/spawnSync, never mutated on process.env.
*
* Every launch writes TWO ledger rows sharing a run_id: a `running`
* reservation (cost = estimate) BEFORE the command starts and a `done`
* reconciliation after it exits. `doneRows()` is the recorded outcome;
* `ledgerRows()` is the raw file.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { spawn, spawnSync } from 'node:child_process';
import {
mkdtempSync,
mkdirSync,
rmSync,
readFileSync,
writeFileSync,
appendFileSync,
chmodSync,
existsSync,
readdirSync,
renameSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
const ROOT = process.cwd();
const SCRIPT = join(ROOT, 'scripts/eval-spend-guard.sh');
// A chmod-444 file is still appendable under CAP_DAC_OVERRIDE (root, some
// sandboxes); the unwritable-ledger pin needs the kernel to honor the bits.
function permissionsEnforced(): boolean {
const d = mkdtempSync(join(tmpdir(), 'gbrain-spend-guard-probe-'));
try {
const f = join(d, 'ro');
writeFileSync(f, '');
chmodSync(f, 0o444);
try {
appendFileSync(f, 'x');
return false;
} catch {
return true;
}
} finally {
rmSync(d, { recursive: true, force: true });
}
}
const PERMS_ENFORCED = permissionsEnforced();
// Same idea for READ bits: a chmod-000 file is still readable under
// CAP_DAC_OVERRIDE; the unreadable-ledger pin needs the kernel to honor it.
function readPermissionsEnforced(): boolean {
const d = mkdtempSync(join(tmpdir(), 'gbrain-spend-guard-probe-'));
try {
const f = join(d, 'noread');
writeFileSync(f, '{"cost_usd":1}\n');
chmodSync(f, 0o000);
try {
readFileSync(f);
return false;
} catch {
return true;
}
} finally {
rmSync(d, { recursive: true, force: true });
}
}
const READ_PERMS_ENFORCED = readPermissionsEnforced();
let dir: string;
let ledger: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'gbrain-spend-guard-'));
ledger = join(dir, 'receipts', 'spend.jsonl');
// The guard requires the ledger to EXIST (a missing ledger is not a $0
// ledger). Every test starts from an empty, present ledger; the missing-
// ledger behaviour has its own tests below, which remove it first.
mkdirSync(join(dir, 'receipts'), { recursive: true });
writeFileSync(ledger, '');
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
function guardEnv(extraEnv: Record<string, string> = {}): Record<string, string> {
return {
PATH: process.env.PATH ?? '/usr/bin:/bin',
HOME: dir,
TMPDIR: dir,
GBRAIN_EVAL_SPEND_LEDGER: ledger,
...extraEnv,
};
}
function run(args: string[], extraEnv: Record<string, string> = {}) {
return spawnSync('bash', [SCRIPT, ...args], { cwd: ROOT, encoding: 'utf-8', env: guardEnv(extraEnv) });
}
/** Async launch for the in-flight / signal pins. */
function start(args: string[], extraEnv: Record<string, string> = {}) {
const child = spawn('bash', [SCRIPT, ...args], { cwd: ROOT, env: guardEnv(extraEnv), stdio: ['ignore', 'pipe', 'pipe'] });
let stderr = '';
child.stderr.on('data', (d: Buffer) => {
stderr += d.toString();
});
const exit = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
child.on('exit', (code, signal) => resolve({ code, signal }));
});
return { child, exit, stderr: () => stderr };
}
async function waitFor(pred: () => boolean, what: string, ms = 8000): Promise<void> {
const t0 = Date.now();
while (!pred()) {
if (Date.now() - t0 > ms) throw new Error(`timed out waiting for ${what}`);
await new Promise((r) => setTimeout(r, 25));
}
}
function ledgerRows(): Array<Record<string, unknown>> {
if (!existsSync(ledger)) return [];
return readFileSync(ledger, 'utf-8')
.split('\n')
.filter((l) => l.trim().length > 0)
.map((l) => JSON.parse(l) as Record<string, unknown>);
}
const doneRows = () => ledgerRows().filter((r) => r.status === 'done');
const runningRows = () => ledgerRows().filter((r) => r.status === 'running');
describe('eval-spend-guard.sh', () => {
test('under cap: runs the command; reservation row then reconciliation row share a run_id', () => {
const marker = join(dir, 'ran.txt');
const r = run(['75', '3', '--', 'sh', '-c', `echo ok > "${marker}"`]);
expect(r.status).toBe(0);
expect(existsSync(marker)).toBe(true);
expect(r.stderr).toContain('launching');
expect(r.stderr).toContain('reserved $3.000000');
expect(r.stderr).toContain('recorded cost $3.000000 (exit 0); ledger now $3.000000');
const rows = ledgerRows();
expect(rows.length).toBe(2);
const [reserved, done] = rows;
expect(reserved.status).toBe('running');
expect(reserved.estimate_usd).toBe(3);
expect(reserved.cost_usd).toBe(3);
expect(reserved.exit_code).toBeNull();
expect(typeof reserved.run_id).toBe('string');
expect(String(reserved.run_id).length).toBeGreaterThan(8);
expect(done.status).toBe('done');
expect(done.run_id).toBe(reserved.run_id);
expect(done.estimate_usd).toBe(3);
expect(done.cost_usd).toBe(3);
expect(done.exit_code).toBe(0);
for (const row of rows) {
expect(String(row.command)).toContain('echo ok');
expect(String(row.ts)).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/);
}
});
test('over cap: refuses with exit 3, never runs the command, appends nothing', () => {
// seed the ledger at $73.50 across two rows (one with an exponent form)
writeFileSync(
ledger,
'{"ts":"2026-09-04T00:00:00Z","estimate_usd":70,"cost_usd":70.25,"exit_code":0,"command":"x"}\n' +
'{"ts":"2026-09-04T00:00:01Z","estimate_usd":3,"cost_usd":3.25e0,"exit_code":0,"command":"y"}\n',
);
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '2', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(r.stderr).toContain('REFUSED');
expect(r.stderr).toContain('75.5');
expect(ledgerRows().length).toBe(2);
});
test('exactly at cap is allowed (ledger + estimate == cap)', () => {
writeFileSync(ledger, '{"cost_usd":72}\n');
const r = run(['75', '3', '--', 'true']);
expect(r.status).toBe(0);
expect(ledgerRows().length).toBe(3);
});
test('actual-cost file written by the command overrides the estimate (bare number)', () => {
const costFile = join(dir, 'actual.txt');
const r = run(['75', '3', '--', 'sh', '-c', `printf '1.2345\\n' > "$GBRAIN_EVAL_ACTUAL_COST_FILE"`], {
GBRAIN_EVAL_ACTUAL_COST_FILE: costFile,
});
expect(r.status).toBe(0);
const [done] = doneRows();
expect(done.estimate_usd).toBe(3);
expect(done.cost_usd).toBe(1.2345);
expect(runningRows()[0].cost_usd).toBe(3); // the reservation keeps the estimate
});
test('actual-cost file as JSON {cost_usd} is honored; unset env gets a scratch path exported to the child', () => {
const r = run([
'75',
'3',
'--',
'sh',
'-c',
`test -n "$GBRAIN_EVAL_ACTUAL_COST_FILE" && printf '{"cost_usd": 0.5, "note":"x"}' > "$GBRAIN_EVAL_ACTUAL_COST_FILE"`,
]);
expect(r.status).toBe(0);
expect(doneRows()[0].cost_usd).toBe(0.5);
});
test('command failure: exit code propagates and is recorded', () => {
const r = run(['75', '1', '--', 'sh', '-c', 'exit 7']);
expect(r.status).toBe(7);
expect(doneRows()[0].exit_code).toBe(7);
});
test('command text with quotes and backslashes yields valid JSON', () => {
const r = run(['75', '1', '--', 'sh', '-c', 'echo "a \\"b\\" \\\\ c"']);
expect(r.status).toBe(0);
const rows = ledgerRows(); // JSON.parse would have thrown on a bad line
expect(rows.length).toBe(2);
expect(String(rows[0].command)).toContain('echo');
});
test('usage errors exit 2 (missing --, non-numeric cap)', () => {
expect(run(['75', '1', 'true']).status).toBe(2);
expect(run(['abc', '1', '--', 'true']).status).toBe(2);
expect(run(['75', 'x', '--', 'true']).status).toBe(2);
expect(ledgerRows().length).toBe(0);
});
test('ledger rows accumulate across runs and drive the next decision', () => {
expect(run(['10', '6', '--', 'true']).status).toBe(0);
expect(run(['10', '3', '--', 'true']).status).toBe(0);
const r = run(['10', '2', '--', 'true']); // 9 + 2 > 10
expect(r.status).toBe(3);
expect(ledgerRows().length).toBe(4);
});
// ── reservation / reconciliation (fail closed in TIME) ──────────────────
test('the reservation is on the ledger while the command runs, blocks a concurrent guard, and is superseded by the reconciliation', async () => {
const g = start(['10', '6', '--', 'sh', '-c', 'sleep 2; printf 2 > "$GBRAIN_EVAL_ACTUAL_COST_FILE"']);
await waitFor(() => runningRows().length === 1, 'the reservation row');
const [reserved] = ledgerRows();
expect(ledgerRows().length).toBe(1);
expect(reserved.status).toBe('running');
expect(reserved.cost_usd).toBe(6);
expect(reserved.exit_code).toBeNull();
// A concurrent guard sees $6 in flight: 6 + 5 > 10 → refused, nothing run.
const marker = join(dir, 'should-not-exist.txt');
const c = run(['10', '5', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(c.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(c.stderr).toContain('ledger $6.000000 + estimate $5.000000');
expect(c.stderr).toContain('1 in-flight reservation(s)');
const { code } = await g.exit;
expect(code).toBe(0);
const rows = ledgerRows();
expect(rows.length).toBe(2);
expect(rows[1].status).toBe('done');
expect(rows[1].run_id).toBe(reserved.run_id);
expect(rows[1].cost_usd).toBe(2);
expect(rows[1].exit_code).toBe(0);
expect(g.stderr()).toContain('ledger now $2.000000');
// Reconciled total is $2 (NOT 6 + 2): 2 + 8 == 10 is allowed.
const n = run(['10', '8', '--', 'true']);
expect(n.status).toBe(0);
expect(n.stderr).toContain('ledger $2.000000 (2 row(s))');
expect(n.stderr).not.toContain('in-flight');
});
test('no command substitution is parsed between arming the signal traps and waiting on the command (#8717)', () => {
// A TERM that lands while bash 5.2 parses a top-level $(...) runs the trap inside that parse, where it fails and the
// signal is lost: the guard then waits out its command and exits 0 instead of 143.
const text = readFileSync(SCRIPT, 'utf-8');
const start = text.indexOf("\ntrap 'on_signal 2' INT\n");
const armed = text.slice(start, text.indexOf('\nwait "$CHILD"\n', start));
expect(armed.length).toBeGreaterThan(0);
const code = armed.split('\n').filter((line) => !line.trimStart().startsWith('#')).join('\n');
expect(code.match(/\$\((?!\()|`/g) ?? []).toEqual([]);
});
test('a SIGTERMed guard stops the command and reconciles at the estimate (exit 143)', async () => {
const g = start(['10', '4', '--', 'sleep', '30']);
// Deterministic: the `reserved $` line is printed AFTER the reservation
// row lands and the traps are armed, so the signal can never race the
// guard's own setup (the pre-fix flake: traps were installed after the
// append, and a TERM in that window left the reservation unreconciled).
await waitFor(() => g.stderr().includes('reserved $4.000000'), "the 'reserved $' line");
expect(runningRows().length).toBe(1);
const t0 = Date.now();
g.child.kill('SIGTERM');
const { code } = await g.exit;
expect(code).toBe(143);
expect(Date.now() - t0).toBeLessThan(10_000); // the child was killed, not waited out
expect(g.stderr()).toContain('interrupted (signal 15)');
const rows = ledgerRows();
expect(rows.length).toBe(2);
expect(rows[1].status).toBe('done');
expect(rows[1].run_id).toBe(rows[0].run_id);
expect(rows[1].cost_usd).toBe(4); // the estimate
expect(rows[1].exit_code).toBe(143);
// …and that spend drives the next decision: 4 + 7 > 10.
expect(run(['10', '7', '--', 'true']).status).toBe(3);
expect(run(['10', '6', '--', 'true']).status).toBe(0);
});
test('a SIGKILLed guard leaves its reservation counted at the estimate for the next guard', async () => {
const g = start(['10', '4', '--', 'sleep', '3']);
await waitFor(() => runningRows().length === 1, 'the reservation row');
g.child.kill('SIGKILL');
const { signal } = await g.exit;
expect(signal).toBe('SIGKILL');
expect(ledgerRows().length).toBe(1); // no reconciliation could be written
const r = run(['10', '7', '--', 'true']); // 4 (in flight forever) + 7 > 10
expect(r.status).toBe(3);
expect(r.stderr).toContain('1 in-flight reservation(s)');
const ok = run(['10', '6', '--', 'true']);
expect(ok.status).toBe(0);
expect(ok.stderr).toContain('1 in-flight reservation(s) counted at their estimate');
});
test('a command that ignores SIGTERM is SIGKILLed after the grace period; the guard still reconciles (exit 143)', async () => {
// The child traps TERM and would sleep on forever; with the grace knob at
// 1 s the guard escalates to KILL instead of waiting unbounded.
const g = start(['10', '4', '--', 'bash', '-c', 'trap "" TERM; sleep 30'], { GBRAIN_EVAL_SPEND_GUARD_KILL_GRACE_SECONDS: '1' });
await waitFor(() => g.stderr().includes('reserved $4.000000'), "the 'reserved $' line");
// Let the child install its trap before we signal.
await new Promise((r) => setTimeout(r, 300));
const t0 = Date.now();
g.child.kill('SIGTERM');
const { code } = await g.exit;
const elapsed = Date.now() - t0;
expect(code).toBe(143);
expect(elapsed).toBeGreaterThanOrEqual(900);
expect(elapsed).toBeLessThan(8_000); // not the child's 30 s
expect(g.stderr()).toContain('did not exit within 1s of SIGTERM — sending SIGKILL');
const rows = ledgerRows();
expect(rows.length).toBe(2);
expect(rows[1].status).toBe('done');
expect(rows[1].cost_usd).toBe(4);
expect(rows[1].exit_code).toBe(143);
}, 15_000);
test.skipIf(!READ_PERMS_ENFORCED)('an existing but unreadable ledger refuses the launch (exit 3) instead of auditing as $0', () => {
// gawk exits 0 with an all-zero END line on a file it cannot open — pre-fix
// that read as "$0 spent" and the launch proceeded against an empty audit.
writeFileSync(ledger, '{"cost_usd":70}\n');
chmodSync(ledger, 0o000);
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(r.stderr).toContain('cannot audit ledger');
expect(r.stderr).toContain('not readable');
expect(r.stderr).toContain('command NOT run');
expect(r.stderr).not.toContain('launching');
chmodSync(ledger, 0o644);
expect(ledgerRows().length).toBe(1); // nothing appended
});
test('an audit awk cannot vouch for (non-zero exit, or a malformed audit line) refuses the launch — never a $0 read', () => {
// Perms-independent twin of the unreadable-ledger pin: a PATH shim replaces
// awk ONLY for the ledger-audit call (every other awk use — norm6, the cap
// arithmetic — delegates to the real binary).
const realAwk = spawnSync('bash', ['-c', 'command -v awk'], { encoding: 'utf-8' }).stdout.trim();
expect(realAwk.length).toBeGreaterThan(0);
const shimDir = join(dir, 'shim');
mkdirSync(shimDir);
const shim = (body: string) => {
writeFileSync(
join(shimDir, 'awk'),
`#!/bin/bash\nfor a in "$@"; do case "$a" in *spend.jsonl) ${body} ;; esac; done\nexec "${realAwk}" "$@"\n`,
);
chmodSync(join(shimDir, 'awk'), 0o755);
};
writeFileSync(ledger, '{"cost_usd":70}\n');
const marker = join(dir, 'should-not-exist.txt');
const env = { PATH: `${shimDir}:${process.env.PATH ?? '/usr/bin:/bin'}` };
// awk dies mid-audit (e.g. an I/O error): refused, nothing appended.
shim('exit 2');
const r1 = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`], env);
expect(r1.status).toBe(3);
expect(r1.stderr).toContain('cannot audit ledger');
expect(r1.stderr).toContain('awk exited 2');
expect(existsSync(marker)).toBe(false);
// awk exits 0 but prints an EMPTY line (the gawk unreadable-file shape).
shim('echo ""; exit 0');
const r2 = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`], env);
expect(r2.status).toBe(3);
expect(r2.stderr).toContain('audit line is malformed');
expect(r2.stderr).not.toContain('launching');
// …or a non-numeric total.
shim('echo "1 1 NaN 0"; exit 0');
const r3 = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`], env);
expect(r3.status).toBe(3);
expect(r3.stderr).toContain('audit line is malformed');
expect(existsSync(marker)).toBe(false);
expect(ledgerRows().length).toBe(1); // untouched throughout
// Control: the real awk through the same PATH still launches (70 + 1 <= 75).
rmSync(join(shimDir, 'awk'));
const ok = run(['75', '1', '--', 'true'], env);
expect(ok.status).toBe(0);
expect(ok.stderr).toContain('ledger $70.000000 (1 row(s))');
});
test('command text with tabs, CR and other control characters is escaped into valid JSON (no GNU-sed \\t dependency)', () => {
const r = run(['75', '1', '--', 'sh', '-c', 'true # tab\there cr\rhere bell\u0001here']);
expect(r.status).toBe(0);
const raw = readFileSync(ledger, 'utf-8');
expect(raw).toContain('tab\\there');
expect(raw).toContain('cr\\rhere');
expect(raw).toContain('bell\\u0001here');
// No raw control byte reached the file …
expect(/[\u0000-\u001f]/.test(raw.replace(/\n/g, ''))).toBe(false);
// … and JSON.parse round-trips the exact command text.
const rows = ledgerRows();
expect(rows.length).toBe(2);
for (const row of rows) expect(String(row.command)).toBe('sh -c true # tab\there cr\rhere bell\u0001here');
});
test('audit: legacy rows are final, an orphan reservation counts, a reconciled reservation does not, a run_id-less running row is final', () => {
writeFileSync(
ledger,
'{"cost_usd":3}\n' + // legacy row (no run_id/status)
'{"run_id":"dead-guard","status":"running","estimate_usd":4,"cost_usd":4,"exit_code":null}\n' +
'{"run_id":"b","status":"running","estimate_usd":9,"cost_usd":9,"exit_code":null}\n' +
'{"run_id":"b","status":"done","estimate_usd":9,"cost_usd":1,"exit_code":0}\n' +
'{"status":"running","cost_usd":0.5}\n', // can never be reconciled → final
);
// 3 + 4 + 1 + 0.5 = 8.5; the $9 reservation for run b is superseded.
const r = run(['10', '1.5', '--', 'true']);
expect(r.status).toBe(0);
expect(r.stderr).toContain('ledger $8.500000 (5 row(s))');
expect(r.stderr).toContain('1 in-flight reservation(s)');
expect(run(['10', '1', '--', 'true']).status).toBe(3); // 10 + 1 > 10
});
test.skipIf(!PERMS_ENFORCED)('an unwritable ledger refuses the launch (exit 3): a run that is not on the books cannot be capped', () => {
writeFileSync(ledger, '{"cost_usd":1}\n');
chmodSync(ledger, 0o444);
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(r.stderr).toContain('cannot append the reservation row');
expect(r.stderr).toContain('command NOT run');
expect(ledgerRows().length).toBe(1); // unchanged
});
// ── fail-closed ledger integrity ─────────────────────────────────────────
test('a truncated ledger line refuses to launch (exit 3) naming the line, and never runs the command', () => {
writeFileSync(
ledger,
'{"ts":"2026-09-04T00:00:00Z","estimate_usd":3,"cost_usd":3.25,"exit_code":0,"command":"x"}\n' +
'{"ts":"2026-09-04T00:00:01Z","estimate_usd":70,"cost_usd":70\n', // killed mid-write: no closing brace
);
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(r.stderr).toContain('unparseable line(s)');
expect(r.stderr).toContain('line numbers: 2');
// nothing appended
expect(readFileSync(ledger, 'utf-8').split('\n').filter((l) => l.length > 0).length).toBe(2);
});
test('a string-typed cost_usd is unparseable, not silently $0 (exit 3, no launch)', () => {
// Pre-fix this row was dropped by the grep and the ledger read as $3.25 —
// fail OPEN. The 70 in the string row is exactly the spend that would blow the cap.
writeFileSync(ledger, '{"cost_usd":3.25}\n{"cost_usd":"70"}\n{"cost_usd":1}\n');
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(r.stderr).toContain('line numbers: 2');
});
test('a signed / negative ledger cost is unparseable too (a negative row would drive the total backwards)', () => {
writeFileSync(ledger, '{"cost_usd":-70}\n{"cost_usd":+1}\n{"cost_usd":2}\n');
const r = run(['75', '1', '--', 'true']);
expect(r.status).toBe(3);
expect(r.stderr).toContain('2 unparseable line(s) out of 3');
expect(r.stderr).toContain('line numbers: 1,2');
});
test('blank lines are tolerated; a valid ledger with whitespace and exponent forms sums exactly', () => {
writeFileSync(ledger, '\n{"cost_usd": 1.5 , "x":1}\n\n{ "cost_usd":2.5e0}\n \n');
const r = run(['5', '1', '--', 'true']);
expect(r.status).toBe(0);
expect(r.stderr).toContain('ledger $4.000000 (2 row(s))');
expect(run(['5', '1', '--', 'true']).status).toBe(3); // 5 + 1 > 5
});
// ── unsigned-only amounts + %.6f normalization ──────────────────────────
test('signed or malformed estimate / cap is a usage error (exit 2), nothing launches, nothing is written', () => {
const marker = join(dir, 'should-not-exist.txt');
for (const [cap, est] of [
['75', '-1'],
['75', '+1'],
['-75', '1'],
['75', '1e'],
['75', 'e5'],
['75', '1.2.3'],
['75', 'NaN'],
['75', 'inf'],
]) {
const r = run([cap, est, '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status, `${cap} ${est}`).toBe(2);
expect(r.stderr, `${cap} ${est}`).toContain('not an unsigned number');
}
expect(existsSync(marker)).toBe(false);
expect(ledgerRows().length).toBe(0);
});
test("estimate forms '1.', '.5' and '2e-1' are accepted and written as valid JSON numbers", () => {
expect(run(['75', '1.', '--', 'true']).status).toBe(0);
expect(run(['75', '.5', '--', 'true']).status).toBe(0);
expect(run(['75', '2e-1', '--', 'true']).status).toBe(0);
const rows = doneRows(); // JSON.parse would throw on `1.` or `.5`
expect(rows.map((r) => r.estimate_usd)).toEqual([1, 0.5, 0.2]);
expect(rows.map((r) => r.cost_usd)).toEqual([1, 0.5, 0.2]);
const raw = readFileSync(ledger, 'utf-8');
expect(raw).toContain('"estimate_usd":1.000000,"cost_usd":1.000000');
expect(raw).toContain('"estimate_usd":0.500000,"cost_usd":0.500000');
// …and the next decision sums them (1.7) correctly.
const r = run(['2', '0.5', '--', 'true']); // 1.7 + 0.5 > 2
expect(r.status).toBe(3);
expect(r.stderr).toContain('ledger $1.700000');
});
test("a cost file of '1.' / '.5' is normalized before it is written", () => {
const costFile = join(dir, 'actual.txt');
const r = run(['75', '3', '--', 'sh', '-c', `printf '.5' > "$GBRAIN_EVAL_ACTUAL_COST_FILE"`], {
GBRAIN_EVAL_ACTUAL_COST_FILE: costFile,
});
expect(r.status).toBe(0);
expect(readFileSync(ledger, 'utf-8')).toContain('"cost_usd":0.500000');
expect(doneRows()[0].cost_usd).toBe(0.5);
});
test('a negative, signed, zero, or string-typed actual cost falls back to the estimate', () => {
const cases: Array<[string, string]> = [
['-2', 'unreadable'], // signed bare number: rejected by is_number, not parsed as -2
['+2', 'unreadable'],
['0', 'non-positive'],
['{"cost_usd":"2"}', 'unreadable'],
['{"cost_usd":-0.5}', 'unreadable'],
['{"cost_usd":0}', 'non-positive'],
['garbage', 'unreadable'],
];
for (const [payload, note] of cases) {
writeFileSync(ledger, '');
const costFile = join(dir, 'actual.txt');
const r = run(['75', '3', '--', 'sh', '-c', `printf '%s' '${payload}' > "$GBRAIN_EVAL_ACTUAL_COST_FILE"`], {
GBRAIN_EVAL_ACTUAL_COST_FILE: costFile,
});
expect(r.status, payload).toBe(0);
expect(r.stderr, payload).toContain(note);
expect(r.stderr, payload).toContain('estimate');
const rows = doneRows();
expect(rows.length, payload).toBe(1);
expect(rows[0].cost_usd, payload).toBe(3); // the estimate, never a negative or zero row
rmSync(costFile, { force: true });
}
});
// ── ledger existence ─────────────────────────────────────────────────────
test('a missing ledger refuses to launch (exit 3) and prints the resolved path', () => {
rmSync(ledger, { force: true });
const marker = join(dir, 'should-not-exist.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo no > "${marker}"`]);
expect(r.status).toBe(3);
expect(existsSync(marker)).toBe(false);
expect(existsSync(ledger)).toBe(false); // NOT silently created
expect(r.stderr).toContain('ledger does not exist');
expect(r.stderr).toContain(ledger);
expect(r.stderr).toContain('GBRAIN_EVAL_SPEND_LEDGER_INIT=1');
});
test('the default ledger path (no env override) is also required to exist', () => {
rmSync(ledger, { force: true });
const r = spawnSync('bash', [SCRIPT, '75', '1', '--', 'true'], {
cwd: ROOT,
encoding: 'utf-8',
env: { PATH: process.env.PATH ?? '/usr/bin:/bin', HOME: dir, TMPDIR: dir },
});
expect(r.status).toBe(3);
expect(r.stderr).toContain(join(dir, 'gbrain-lme-receipts', 'spend.jsonl'));
});
test('GBRAIN_EVAL_SPEND_LEDGER_INIT=1 creates the ledger with a loud NEW LEDGER line, then launches', () => {
rmSync(join(dir, 'receipts'), { recursive: true, force: true });
const marker = join(dir, 'ran.txt');
const r = run(['75', '1', '--', 'sh', '-c', `echo ok > "${marker}"`], { GBRAIN_EVAL_SPEND_LEDGER_INIT: '1' });
expect(r.status).toBe(0);
expect(existsSync(marker)).toBe(true);
expect(r.stderr).toContain('NEW LEDGER');
expect(r.stderr).toContain(ledger);
expect(ledgerRows().length).toBe(2);
// A second run with INIT still set does NOT re-announce (the file exists now).
const r2 = run(['75', '1', '--', 'true'], { GBRAIN_EVAL_SPEND_LEDGER_INIT: '1' });
expect(r2.status).toBe(0);
expect(r2.stderr).not.toContain('NEW LEDGER');
expect(ledgerRows().length).toBe(4);
});
});
// ---------------------------------------------------------------------------
// Process groups (#6776). The guarded command runs in its own process group:
// a signal, the kill grace or a normal exit stops every process in it.
// ---------------------------------------------------------------------------
/** Alive and not a zombie (an orphan's zombie is reaped by init shortly after). */
function alive(pid: number): boolean {
if (!pid) return false;
const state = spawnSync('ps', ['-o', 'stat=', '-p', String(pid)], { encoding: 'utf-8' }).stdout.trim();
return state !== '' && !state.startsWith('Z');
}
async function waitDead(pid: number, ms = 5000): Promise<boolean> {
const t0 = Date.now();
while (alive(pid)) {
if (Date.now() - t0 > ms) return false;
await new Promise((r) => setTimeout(r, 50));
}
return true;
}
function readPid(file: string): number {
return Number(readFileSync(file, 'utf-8').trim());
}
function killQuietly(...pids: number[]): void {
for (const pid of pids) {
if (!pid) continue;
try { process.kill(-pid, 'SIGKILL'); } catch { /* not a group or gone */ }
try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ }
}
}
/** A TERM-ignoring grandchild that writes its pid, under a leader that writes its own. */
function groupCommand(leaderPid: string, kidPid: string, leaderSeconds = 30): string[] {
return ['sh', '-c', 'echo $$ > "' + leaderPid + '"; sh -c \'trap "" TERM; echo $$ > "' + kidPid + '"; exec sleep 30\' & exec sleep ' + leaderSeconds];
}
describe('eval-spend-guard.sh process groups (#6776)', () => {
test('a SIGTERMed guard stops the whole process group, including a grandchild that ignores SIGTERM (exit 143)', async () => {
const leaderFile = join(dir, 'leader.pid');
const kidFile = join(dir, 'kid.pid');
const g = start(['10', '4', '--', ...groupCommand(leaderFile, kidFile)], { GBRAIN_EVAL_SPEND_GUARD_KILL_GRACE_SECONDS: '1' });
let kid = 0;
let leader = 0;
try {
await waitFor(() => existsSync(kidFile) && existsSync(leaderFile), 'the command pids');
await new Promise((r) => setTimeout(r, 200));
kid = readPid(kidFile);
leader = readPid(leaderFile);
g.child.kill('SIGTERM');
const { code } = await g.exit;
expect(code).toBe(143);
expect(await waitDead(leader)).toBe(true);
expect(await waitDead(kid)).toBe(true);
expect(doneRows()[0]?.exit_code).toBe(143);
} finally {
killQuietly(kid, leader);
}
}, 20_000);
test('a command that exits normally but leaves a helper in its group: the helper is stopped when the guard settles', async () => {
const helperFile = join(dir, 'helper.pid');
// Async: the leftover helper holds the guard's stdout/stderr open, so a
// spawnSync would wait for the pipes rather than the guard.
const g = start(['10', '1', '--', 'sh', '-c', 'sleep 30 & echo $! > "' + helperFile + '"; exit 0'], { GBRAIN_EVAL_SPEND_GUARD_KILL_GRACE_SECONDS: '1' });
const { code } = await g.exit;
const helper = existsSync(helperFile) ? readPid(helperFile) : 0;
try {
expect(code).toBe(0);
expect(helper).toBeGreaterThan(0);
expect(await waitDead(helper)).toBe(true);
expect(doneRows()[0]?.exit_code).toBe(0);
} finally {
killQuietly(helper);
}
}, 20_000);
test('piped stdin still reaches the command', () => {
const r = spawnSync('bash', [SCRIPT, '5', '1', '--', 'sh', '-c', 'read value; printf "received:%s" "$value"'], {
cwd: ROOT, env: guardEnv(), input: 'example-input\n', encoding: 'utf-8', timeout: 15_000,
});
expect(r.status).toBe(0);
expect(r.stdout).toContain('received:example-input');
});
test.skipIf(process.platform === 'win32')('terminal stdin becomes EOF with a notice instead of a command stuck reading the terminal', async () => {
let screen = '';
let proc: ReturnType<typeof Bun.spawn> | undefined;
try {
proc = Bun.spawn(['bash', SCRIPT, '5', '1', '--', 'sh', '-c', 'if read value; then echo READ-LINE; else echo READ-EOF; fi'], {
cwd: ROOT,
env: guardEnv(),
terminal: { cols: 200, rows: 20, data(_t: unknown, d: Uint8Array) { screen += new TextDecoder().decode(d); } },
} as unknown as Parameters<typeof Bun.spawn>[1]);
const exit = await Promise.race([proc.exited, new Promise<'timeout'>((r) => setTimeout(() => r('timeout'), 8000))]);
expect(exit).toBe(0);
expect(screen).toContain('READ-EOF');
expect(screen).toContain('stdin is a terminal');
} finally {
proc?.kill('SIGKILL');
}
}, 15_000);
});
// ---------------------------------------------------------------------------
// Stale run leases (r3-d1): a guard records its run's lease while the command
// runs; a later start REPORTS a run whose guard is gone but whose group may
// still run (inspect/stop commands, an unknown ledger row) and kills nothing.
// ---------------------------------------------------------------------------
const leaseDir = () => ledger + '.leases';
const leaseFiles = () => (existsSync(leaseDir()) ? readdirSync(leaseDir()) : []);
const unknownRows = () => ledgerRows().filter((r) => r.status === 'unknown');
function writeLease(name: string, fields: Record<string, string | number>): string {
mkdirSync(leaseDir(), { recursive: true });
const file = join(leaseDir(), name + '.lease');
writeFileSync(file, Object.entries(fields).map(([k, v]) => k + '=' + v).join('\n') + '\n');
return file;
}
describe('eval-spend-guard.sh stale run leases (r3-d1)', () => {
test('a run records its lease (pgid, host, boot id) while it runs and removes it once reconciled', async () => {
const g = start(['10', '1', '--', 'sleep', '30']);
try {
await waitFor(() => leaseFiles().some((f) => f.endsWith('.lease')), 'the lease file');
const body = readFileSync(join(leaseDir(), leaseFiles()[0]!), 'utf-8');
expect(body).toContain('run_id=' + String(runningRows()[0]!.run_id));
expect(body).toMatch(/\nguard_pid=\d+\n/);
expect(body).toMatch(/\npgid=\d+\n/);
expect(body).toMatch(/\nhost=.+\n/);
expect(body).toMatch(/\nboot_id=.+\n/);
} finally {
g.child.kill('SIGTERM');
await g.exit;
}
expect(leaseFiles()).toEqual([]);
expect(run(['10', '1', '--', 'true']).status).toBe(0);
expect(leaseFiles()).toEqual([]);
}, 20_000);
test("a SIGKILLed guard's still-running group is reported with inspect and stop commands and recorded as unknown; nothing is killed and the reservation stays counted", async () => {
const leaderFile = join(dir, 'leader.pid');
const kidFile = join(dir, 'kid.pid');
const g = start(['10', '4', '--', ...groupCommand(leaderFile, kidFile)]);
let kid = 0;
let leader = 0;
try {
await waitFor(() => existsSync(kidFile) && existsSync(leaderFile) && leaseFiles().length === 1, 'the command pids and lease');
kid = readPid(kidFile);
leader = readPid(leaderFile);
const runId = String(runningRows()[0]!.run_id);
const lease = join(leaseDir(), runId + '.lease');
g.child.kill('SIGKILL');
await g.exit;
expect(alive(leader)).toBe(true);
const next = run(['10', '1', '--', 'true']);
expect(next.status).toBe(0);
expect(next.stderr).toContain('STALE RUN ' + runId);
expect(next.stderr).toContain('Lease: ' + lease);
expect(next.stderr).toContain('ps -o pid,pgid,cmd -g ' + leader);
expect(next.stderr).toContain('kill -TERM -- -' + leader);
expect(next.stderr).toContain('nothing was stopped');
expect(next.stderr).toContain('retired automatically once the group exits (or now: rm ' + lease + '.reported)');
expect(unknownRows()).toEqual([expect.objectContaining({ stale_run_id: runId, pgid: leader, lease, cost_usd: 0, reason: 'guard_gone_group_may_run' })]);
expect(unknownRows()[0]).not.toHaveProperty('run_id');
expect(alive(leader)).toBe(true);
expect(alive(kid)).toBe(true);
expect(leaseFiles()).toEqual([runId + '.lease.reported']);
// The stale reservation (estimate 4) is still counted: 4 + the next run's 1 + 1 = 6 > cap 5 refuses.
const capped = run(['5', '1', '--', 'true']);
expect(capped.status).toBe(3);
expect(capped.stderr).not.toContain('STALE RUN');
expect(unknownRows()).toHaveLength(1);
} finally {
killQuietly(kid, leader);
}
}, 30_000);
test("a lease whose group is gone is removed silently; another host's lease is left alone", () => {
const deadGuard = spawnSync('true').pid;
const deadGroup = spawnSync('true').pid;
const gone = writeLease('gone-run', { run_id: 'gone-run', guard_pid: deadGuard, pgid: deadGroup, host: spawnSync('hostname', { encoding: 'utf-8' }).stdout.trim(), boot_id: 'unknown' });
const foreign = writeLease('foreign-run', { run_id: 'foreign-run', guard_pid: deadGuard, pgid: process.pid, host: 'host-b.example', boot_id: 'unknown' });
const r = run(['10', '1', '--', 'true']);
expect(r.status).toBe(0);
expect(r.stderr).not.toContain('STALE RUN');
expect(existsSync(gone)).toBe(false);
expect(existsSync(foreign)).toBe(true);
expect(unknownRows()).toEqual([]);
});
test('a reported lease is retired on the next start once its group has exited, with no new ledger row; one whose group still runs is kept', async () => {
const host = spawnSync('hostname', { encoding: 'utf-8' }).stdout.trim();
const deadGuard = spawnSync('true').pid;
const deadGroup = spawnSync('true').pid;
const live = spawn('sleep', ['30'], { detached: true, stdio: 'ignore' });
try {
await waitFor(() => alive(live.pid ?? 0), 'the live group');
const exited = writeLease('exited-run', { run_id: 'exited-run', guard_pid: deadGuard, pgid: deadGroup, host, boot_id: 'unknown' }) + '.reported';
renameSync(exited.replace(/\.reported$/, ''), exited);
const running = writeLease('running-run', { run_id: 'running-run', guard_pid: deadGuard, pgid: live.pid!, host, boot_id: 'unknown' }) + '.reported';
renameSync(running.replace(/\.reported$/, ''), running);
const r = run(['10', '1', '--', 'true']);
expect(r.status).toBe(0);
expect(r.stderr).not.toContain('STALE RUN');
expect(existsSync(exited)).toBe(false);
expect(existsSync(running)).toBe(true);
expect(alive(live.pid!)).toBe(true);
expect(unknownRows()).toEqual([]);
} finally {
killQuietly(live.pid ?? 0);
}
}, 20_000);
});