Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #4218 — chat usage accounting (revives the #3392 shape).
*
* Pins the full loop: gateway.chat() success boundary → recordChatUsage →
* phase attribution (direct vs job callers) → canonical cost math incl.
* cache tokens → engine sink INSERT into chat_usage_log (migration v140) →
* the admin-scope get_usage op with explicit coverage fields (admin: the
* ledger has no source dimension, so source-scoped read tokens are fenced). Failure paths:
* a failed chat writes NO row; a throwing sink never breaks a chat call;
* unknown-pricing models record cost_usd NULL (never a fake 0).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import {
__setChatTransportForTests,
__setGenerateTextTransportForTests,
chat,
configureGateway,
resetGateway,
type ChatResult,
} from '../src/core/ai/gateway.ts';
import {
withChatPhase,
withChatCallMeter,
currentChatPhase,
setChatUsageSink,
registerChatUsageSink,
recordChatUsage,
estimateChatCostUsd,
makeEngineChatUsageSink,
type ChatCallMeter,
type ChatUsageRecord,
} from '../src/core/ai/chat-usage.ts';
import { operationsByName, type OperationContext } from '../src/core/operations.ts';
function fakeResult(over: Partial<ChatResult> = {}): ChatResult {
return {
text: 'ok',
blocks: [{ type: 'text', text: 'ok' }],
stopReason: 'end',
usage: { input_tokens: 1000, output_tokens: 500, cache_read_tokens: 0, cache_creation_tokens: 0 },
model: 'anthropic:claude-haiku-4-5',
providerId: 'anthropic',
...over,
};
}
afterEach(() => {
__setChatTransportForTests(null);
setChatUsageSink(null);
});
describe('#4480 — sink registry: multi-engine deregistration restores the prior ledger', () => {
const usage = { input_tokens: 1, output_tokens: 1 };
test('records route to the top live sink; deregistering it restores the previous one', () => {
const primary: ChatUsageRecord[] = [];
const secondary: ChatUsageRecord[] = [];
const deregPrimary = registerChatUsageSink((r) => { primary.push(r); });
const deregSecondary = registerChatUsageSink((r) => { secondary.push(r); });
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(secondary.length).toBe(1);
expect(primary.length).toBe(0);
// The short-lived secondary engine disconnects. Pre-fix (last-wins
// scalar) the ledger was permanently lost to the closed engine; now the
// primary sink is restored.
deregSecondary();
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(primary.length).toBe(1);
expect(secondary.length).toBe(1);
deregPrimary();
// No live sink: record is a silent no-op (never throws).
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(primary.length).toBe(1);
// Deregistration is idempotent.
deregSecondary();
deregPrimary();
});
test('out-of-order deregistration removes only its own entry', () => {
const a: ChatUsageRecord[] = [];
const b: ChatUsageRecord[] = [];
const c: ChatUsageRecord[] = [];
const deregA = registerChatUsageSink((r) => { a.push(r); });
const deregB = registerChatUsageSink((r) => { b.push(r); });
const deregC = registerChatUsageSink((r) => { c.push(r); });
// Middle entry disconnects first — top stays the router.
deregB();
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(c.length).toBe(1);
deregC();
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(a.length).toBe(1);
expect(b.length).toBe(0);
deregA();
});
test('setChatUsageSink(null) clears every registered entry (legacy scalar API)', () => {
const a: ChatUsageRecord[] = [];
registerChatUsageSink((r) => { a.push(r); });
setChatUsageSink(null);
recordChatUsage({ model: 'anthropic:claude-haiku-4-5', usage });
expect(a.length).toBe(0);
});
});
describe('estimateChatCostUsd — canonical pricing incl. cache tokens', () => {
test('anthropic model prices cache reads at 0.1x and writes at 1.25x input', () => {
// haiku-4-5: $1/M in, $5/M out, $0.10/M cache-read, $1.25/M cache-write
const cost = estimateChatCostUsd('anthropic:claude-haiku-4-5', {
input_tokens: 1_000_000,
output_tokens: 1_000_000,
cache_read_tokens: 1_000_000,
cache_write_tokens: 1_000_000,
});
expect(cost).toBeCloseTo(1 + 5 + 0.1 + 1.25, 8);
});
test('provider without cache rates falls back to the input rate for cache tokens', () => {
// gpt-4o: $2.50/M in, $10/M out; no cache fields → cache tokens at input rate
const cost = estimateChatCostUsd('openai:gpt-4o', {
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 1_000_000,
cache_write_tokens: 1_000_000,
});
expect(cost).toBeCloseTo(2.5 + 2.5, 8);
});
test('unknown model → null, never a fake 0', () => {
expect(
estimateChatCostUsd('acme:unpriced-model-9000', { input_tokens: 10, output_tokens: 10 }),
).toBeNull();
});
});
describe('#5506: chat call meter prices what it counts', () => {
const usage = { input_tokens: 1000, output_tokens: 500 };
const PRICED = 'anthropic:claude-haiku-4-5';
const UNPRICED = 'acme:unpriced-model-9000';
function meterCalls(meter: ChatCallMeter, models: string[]): void {
withChatCallMeter(meter, () => {
for (const model of models) recordChatUsage({ model, usage });
});
}
test('a pricing meter sums priced calls and counts unpriced ones apart, with no sink registered', () => {
setChatUsageSink(null);
const meter: ChatCallMeter = { calls: 0, cost_usd: 0, unpriced_calls: 0 };
meterCalls(meter, [PRICED, UNPRICED, PRICED]);
expect(meter.calls).toBe(3);
expect(meter.unpriced_calls).toBe(1);
expect(meter.cost_usd).toBeCloseTo(2 * estimateChatCostUsd(PRICED, usage)!, 12);
});
test('the sink record carries the same cost the meter added', () => {
const records: ChatUsageRecord[] = [];
setChatUsageSink((r) => { records.push(r); });
const meter: ChatCallMeter = { calls: 0, cost_usd: 0, unpriced_calls: 0 };
meterCalls(meter, [PRICED, UNPRICED]);
expect(records.map(r => r.cost_usd)).toEqual([meter.cost_usd!, null]);
});
test('the meter prices with the budget tracker resolver: overrides, the claude-cli sibling rate and free local models', () => {
setChatUsageSink(null);
const meter: ChatCallMeter = { calls: 0, cost_usd: 0, unpriced_calls: 0, pricing_overrides: { [UNPRICED]: { input: 2, output: 4 } } };
meterCalls(meter, [UNPRICED, 'claude-cli:claude-haiku-4-5', 'ollama:llama3']);
expect(meter.unpriced_calls).toBe(0);
const overridden = (usage.input_tokens * 2 + usage.output_tokens * 4) / 1_000_000;
expect(meter.cost_usd).toBeCloseTo(overridden + estimateChatCostUsd('anthropic:claude-haiku-4-5', usage)!, 12);
});
test('a { calls } meter still only counts calls', () => {
const meter: ChatCallMeter = { calls: 0 };
meterCalls(meter, [PRICED, UNPRICED]);
expect(meter).toEqual({ calls: 2 });
});
});
describe('gateway.chat() success boundary — direct vs job callers', () => {
test('direct caller records phase NULL; job caller records job:<name>', async () => {
const records: ChatUsageRecord[] = [];
setChatUsageSink((r) => {
records.push(r);
});
__setChatTransportForTests(async () => fakeResult());
await chat({ model: 'anthropic:claude-haiku-4-5', messages: [{ role: 'user', content: 'hi' }] });
await withChatPhase('job:embed-backfill', () =>
chat({ model: 'anthropic:claude-haiku-4-5', messages: [{ role: 'user', content: 'hi' }] }),
);
// sink is fire-and-forget — let the microtask queue drain
await new Promise((r) => setTimeout(r, 10));
expect(records.length).toBe(2);
expect(records[0]!.phase).toBeNull();
expect(records[1]!.phase).toBe('job:embed-backfill');
expect(records[0]!.model).toBe('anthropic:claude-haiku-4-5');
expect(records[0]!.input_tokens).toBe(1000);
expect(records[0]!.output_tokens).toBe(500);
// $1/M * 1000 + $5/M * 500 = 0.001 + 0.0025
expect(records[0]!.cost_usd).toBeCloseTo(0.0035, 8);
});
test('cache tokens flow through (cache_creation → cache_write)', async () => {
const records: ChatUsageRecord[] = [];
setChatUsageSink((r) => {
records.push(r);
});
__setChatTransportForTests(async () =>
fakeResult({
// ChatResult convention: input_tokens is the total, the cache buckets are inside it (100 uncached).
usage: { input_tokens: 5100, output_tokens: 50, cache_read_tokens: 2000, cache_creation_tokens: 3000 },
}),
);
await chat({ model: 'anthropic:claude-haiku-4-5', messages: [{ role: 'user', content: 'hi' }] });
await new Promise((r) => setTimeout(r, 10));
expect(records[0]!.cache_read_tokens).toBe(2000);
expect(records[0]!.cache_write_tokens).toBe(3000);
expect(records[0]!.cost_usd).toBeCloseTo(
(100 * 1 + 50 * 5 + 2000 * 0.1 + 3000 * 1.25) / 1e6,
10,
);
});
test('failed chat writes NO usage row and the error propagates', async () => {
const records: ChatUsageRecord[] = [];
setChatUsageSink((r) => {
records.push(r);
});
__setChatTransportForTests(async () => {
throw new Error('provider exploded');
});
await expect(
chat({ model: 'anthropic:claude-haiku-4-5', messages: [{ role: 'user', content: 'hi' }] }),
).rejects.toThrow('provider exploded');
await new Promise((r) => setTimeout(r, 10));
expect(records.length).toBe(0);
});
test('a throwing sink never breaks a chat call (fail-open)', async () => {
setChatUsageSink(() => {
throw new Error('sink down');
});
__setChatTransportForTests(async () => fakeResult());
const res = await chat({
model: 'anthropic:claude-haiku-4-5',
messages: [{ role: 'user', content: 'hi' }],
});
expect(res.text).toBe('ok');
});
test('withChatPhase restores the outer phase after the callback', async () => {
expect(currentChatPhase()).toBeNull();
await withChatPhase('job:outer', async () => {
expect(currentChatPhase()).toBe('job:outer');
});
expect(currentChatPhase()).toBeNull();
});
});
describe('engine sink + get_usage op (PGLite, migration v140 schema)', () => {
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await engine.executeRaw('DELETE FROM chat_usage_log');
});
function ctx(over: Partial<OperationContext> = {}): OperationContext {
return {
engine,
config: {} as OperationContext['config'],
logger: { info() {}, warn() {}, error() {}, debug() {} } as unknown as OperationContext['logger'],
dryRun: false,
remote: true,
sourceId: 'default',
...over,
} as OperationContext;
}
test('sink inserts; get_usage aggregates with explicit coverage fields', async () => {
const sink = makeEngineChatUsageSink(engine);
await sink({
model: 'anthropic:claude-haiku-4-5',
provider: 'anthropic',
phase: null,
input_tokens: 1000,
output_tokens: 500,
cache_read_tokens: 200,
cache_write_tokens: 300,
cost_usd: 0.0035,
});
await sink({
model: 'anthropic:claude-haiku-4-5',
provider: 'anthropic',
phase: 'job:embed-backfill',
input_tokens: 2000,
output_tokens: 1000,
cache_read_tokens: 0,
cache_write_tokens: 0,
cost_usd: 0.007,
});
// Unknown-pricing model: tokens counted, dollars NULL.
await sink({
model: 'acme:unpriced-model-9000',
provider: 'acme',
phase: 'job:dream',
input_tokens: 10,
output_tokens: 20,
cache_read_tokens: 0,
cache_write_tokens: 0,
cost_usd: null,
});
const get_usage = operationsByName['get_usage']!;
expect(get_usage.scope).toBe('admin');
const out = (await get_usage.handler(ctx(), { days: 7 })) as any;
expect(out.window_days).toBe(7);
expect(out.totals.calls).toBe(3);
expect(out.totals.input_tokens).toBe(3010);
expect(out.totals.output_tokens).toBe(1520);
expect(out.totals.cache_read_tokens).toBe(200);
expect(out.totals.cache_write_tokens).toBe(300);
expect(out.totals.cost_usd).toBeCloseTo(0.0105, 8);
const haiku = out.by_model.find((m: any) => m.model === 'anthropic:claude-haiku-4-5');
expect(haiku.calls).toBe(2);
expect(haiku.cost_usd).toBeCloseTo(0.0105, 8);
expect(haiku.unpriced_calls).toBe(0);
const unpriced = out.by_model.find((m: any) => m.model === 'acme:unpriced-model-9000');
expect(unpriced.calls).toBe(1);
expect(unpriced.cost_usd).toBeNull();
expect(unpriced.unpriced_calls).toBe(1);
const direct = out.by_phase.find((p: any) => p.phase === 'direct');
expect(direct.calls).toBe(1);
const jobPhase = out.by_phase.find((p: any) => p.phase === 'job:embed-backfill');
expect(jobPhase.calls).toBe(1);
// Coverage honesty contract.
expect(out.coverage.source).toBe('gateway.chat');
expect(out.coverage.table_present).toBe(true);
expect(out.coverage.logged_since).not.toBeNull();
expect(out.coverage.priced_calls).toBe(2);
expect(out.coverage.unpriced_calls).toBe(1);
expect(Array.isArray(out.coverage.not_covered)).toBe(true);
expect(out.coverage.not_covered.length).toBeGreaterThan(0);
});
test('end-to-end: gateway transport call lands a row via the engine sink', async () => {
setChatUsageSink(makeEngineChatUsageSink(engine));
__setChatTransportForTests(async () => fakeResult());
await withChatPhase('job:signal-detector', () =>
chat({ model: 'anthropic:claude-haiku-4-5', messages: [{ role: 'user', content: 'hi' }] }),
);
// The sink INSERT is fire-and-forget; poll briefly for the row.
let rows: Array<{ phase: string | null; cost_usd: number | null }> = [];
for (let i = 0; i < 50 && rows.length === 0; i++) {
await new Promise((r) => setTimeout(r, 20));
rows = await engine.executeRaw(
'SELECT phase, cost_usd FROM chat_usage_log',
[],
);
}
expect(rows.length).toBe(1);
expect(rows[0]!.phase).toBe('job:signal-detector');
expect(Number(rows[0]!.cost_usd)).toBeCloseTo(0.0035, 8);
});
test('empty window → zeroed totals, coverage still explicit', async () => {
const get_usage = operationsByName['get_usage']!;
const out = (await get_usage.handler(ctx(), {})) as any;
expect(out.window_days).toBe(30);
expect(out.totals.calls).toBe(0);
expect(out.by_model).toEqual([]);
expect(out.coverage.table_present).toBe(true);
});
});
// A1 follow-up: the gateway reports AI SDK usage, where the input total
// already contains the cache buckets. A 100-token prompt with 60 tokens read
// from cache must be priced as 40 uncached + 60 cache-read, whichever
// convention the provider used; adding the 60 on top of the 100 (master)
// priced 160 input tokens. The generateText seam replaces only the SDK call,
// so the gateway's own usage assembly runs.
describe('cache tokens are subsets of the gateway input total (both provider conventions)', () => {
afterEach(() => {
__setGenerateTextTransportForTests(null);
resetGateway();
});
const CASES = [
{ model: 'anthropic:claude-haiku-4-5', result: { finishReason: 'stop',
usage: { inputTokens: 100, inputTokenDetails: { noCacheTokens: 40, cacheReadTokens: 60, cacheWriteTokens: 0 }, outputTokens: 10 },
providerMetadata: { anthropic: { cacheReadInputTokens: 60, cacheCreationInputTokens: 0 } } } },
{ model: 'openai:gpt-4o', result: { finishReason: 'stop',
usage: { inputTokens: 100, inputTokenDetails: { noCacheTokens: 40, cacheReadTokens: 60, cacheWriteTokens: undefined }, outputTokens: 10, cachedInputTokens: 60 } } },
] as const;
for (const { model, result } of CASES) {
test(`${model}: 40 uncached + 60 cache-read, not 100 + 60`, async () => {
configureGateway({ env: { ANTHROPIC_API_KEY: 'sk-ant-fake', OPENAI_API_KEY: 'sk-fake' } });
__setGenerateTextTransportForTests((async () => ({ content: [{ type: 'text', text: 'ok' }], ...result })) as never);
const records: ChatUsageRecord[] = [];
setChatUsageSink((r) => { records.push(r); });
await chat({ model, messages: [{ role: 'user', content: 'hi' }] });
await new Promise((r) => setTimeout(r, 10));
expect(records.length).toBe(1);
expect(records[0]!.input_tokens).toBe(100);
expect(records[0]!.cache_read_tokens).toBe(60);
const expected = estimateChatCostUsd(model, { input_tokens: 40, output_tokens: 10, cache_read_tokens: 60 })!;
expect(expected).toBeGreaterThan(0);
expect(records[0]!.cost_usd).toBeCloseTo(expected, 12);
expect(records[0]!.cost_usd).toBeLessThan(estimateChatCostUsd(model, { input_tokens: 100, output_tokens: 10, cache_read_tokens: 60 })!);
});
}
test('a priced meter (non-table override) charges the input total once', () => {
const meter: ChatCallMeter = { calls: 0, cost_usd: 0, pricing_overrides: { 'acme:override-model': { input: 1, output: 0 } } as never };
withChatCallMeter(meter, () => recordChatUsage({ model: 'acme:override-model', usage: { input_tokens: 100, output_tokens: 0, cache_read_tokens: 60 } }));
expect(meter.cost_usd).toBeCloseTo(100 / 1e6, 12);
});
});