Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.4 KB
Raw Blame
/**
 * #5585 — BudgetTracker per-model ledger (`snapshot().models`), the shared
 * `reservationCostUsd` pricing helper, and the `models_used` builders.
 *
 * Hermetic: every tracker writes its audit JSONL to a per-test tempdir; no
 * gateway, provider or network is touched.
 */

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { BudgetExhausted, BudgetTracker } from '../../../src/core/budget/budget-tracker.ts';
import { reservationCostUsd } from '../../../src/core/budget/reservation-cost.ts';
import {
  buildModelsUsed,
  formatModelsUsedTable,
  mergeModelUsageRows,
  touchpointForLabel,
  type ModelUsageRow,
} from '../../../src/core/budget/models-used.ts';

const HAIKU = 'anthropic:claude-haiku-4-5-20251001';

let tmp: string;
let auditPath: string;

beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), 'gbrain-model-ledger-'));
  auditPath = join(tmp, 'budget.jsonl');
});

afterEach(() => {
  rmSync(tmp, { recursive: true, force: true });
});

const tracker = (maxCostUsd?: number) =>
  new BudgetTracker({ label: 'ledger-test', auditPath, ...(maxCostUsd !== undefined ? { maxCostUsd } : {}) });

describe('touchpointForLabel', () => {
  test('maps gateway labels (incl. .failed) and falls back to other', () => {
    expect(touchpointForLabel('gateway.chat')).toBe('chat');
    expect(touchpointForLabel('gateway.expand.failed')).toBe('expansion');
    expect(touchpointForLabel('gateway.embed')).toBe('embedding');
    expect(touchpointForLabel('gateway.rerank')).toBe('rerank');
    expect(touchpointForLabel('gateway.ocr.failed')).toBe('ocr');
    expect(touchpointForLabel('gateway.chatter')).toBe('other');
    expect(touchpointForLabel(undefined)).toBe('other');
  });
});

describe('BudgetTracker ledger', () => {
  test('aggregates per (requested, served, touchpoint, purpose) with tokens and cost', () => {
    const t = tracker();
    t.record({ modelId: HAIKU, inputTokens: 1000, outputTokens: 100, label: 'gateway.chat', purpose: 'skillopt.judge' });
    t.record({ modelId: HAIKU, inputTokens: 500, outputTokens: 50, label: 'gateway.chat', purpose: 'skillopt.judge' });
    const [row, ...rest] = t.snapshot().models;
    expect(rest).toEqual([]);
    expect(row).toEqual({
      requested_model: HAIKU, model: HAIKU, touchpoint: 'chat', purpose: 'skillopt.judge',
      calls: 2, attempts: 2, failed_calls: 0, input_tokens: 1500, output_tokens: 150,
      cost_usd: expect.any(Number), cost_basis: 'measured',
    });
    expect(row!.cost_usd).toBeCloseTo(t.totalSpent, 12);
  });

  test('same model, two purposes -> two rows; engine-internal calls carry purpose null', () => {
    const t = tracker();
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 1, label: 'gateway.chat', purpose: 'skillopt.optimizer' });
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 1, label: 'gateway.chat', purpose: 'skillopt.judge' });
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 1, label: 'gateway.expand' });
    const rows = t.snapshot().models;
    expect(rows.map(r => [r.touchpoint, r.purpose])).toEqual([
      ['chat', 'skillopt.judge'],
      ['chat', 'skillopt.optimizer'],
      ['expansion', null],
    ]);
  });

  test('.failed labels and explicit failed both count as failed calls; estimated sets cost_basis', () => {
    const t = tracker();
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 5, label: 'gateway.embed.failed', kind: 'chat' });
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 5, label: 'gateway.chat', failed: true, estimated: true });
    t.record({ modelId: HAIKU, inputTokens: 10, outputTokens: 5, label: 'gateway.chat' });
    const rows = t.snapshot().models;
    expect(rows.find(r => r.touchpoint === 'chat')).toMatchObject({ calls: 2, failed_calls: 1, cost_basis: 'estimated' });
    expect(rows.find(r => r.touchpoint === 'embedding')).toMatchObject({ calls: 1, failed_calls: 1, cost_basis: 'measured' });
  });

  test('countsAsCall:false adds an attempt, not a call', () => {
    const t = tracker();
    t.record({ modelId: HAIKU, inputTokens: 10, label: 'gateway.expand.failed' });
    t.record({ modelId: HAIKU, inputTokens: 10, label: 'gateway.expand', countsAsCall: false });
    expect(t.snapshot().models).toEqual([expect.objectContaining({ calls: 1, attempts: 2, failed_calls: 1 })]);
  });

  test('unpriced record stays in the ledger with cost_usd null', () => {
    const t = tracker();
    t.record({ modelId: 'litellm:mystery', inputTokens: 42, outputTokens: 7, label: 'gateway.chat' });
    expect(t.totalSpent).toBe(0);
    expect(t.snapshot().models).toEqual([expect.objectContaining({
      model: 'litellm:mystery', calls: 1, input_tokens: 42, output_tokens: 7, cost_usd: null,
    })]);
  });

  test('a TX1 over-cap record throws AND is still in the ledger', () => {
    const t = tracker(0.000001);
    expect(() => t.record({ modelId: HAIKU, inputTokens: 100_000, outputTokens: 10_000, label: 'gateway.chat' }))
      .toThrow(BudgetExhausted);
    expect(t.snapshot().models).toEqual([expect.objectContaining({ calls: 1, input_tokens: 100_000, cost_usd: expect.any(Number) })]);
  });

  test('requested vs served stay distinct; alias and canonical served ids aggregate to one row', () => {
    const t = tracker();
    t.record({ modelId: 'claude-cli:haiku', requestedModelId: 'claude-cli:haiku', inputTokens: 10, label: 'gateway.chat' });
    t.record({ modelId: 'claude-cli:claude-haiku-4-5-20251001', requestedModelId: 'claude-cli:haiku', inputTokens: 10, label: 'gateway.chat' });
    expect(t.snapshot().models).toEqual([expect.objectContaining({
      requested_model: 'claude-cli:haiku', model: 'claude-cli:claude-haiku-4-5-20251001', calls: 2,
    })]);
  });

  test('two trackers keep isolated ledgers', () => {
    const a = tracker();
    const b = tracker();
    a.record({ modelId: HAIKU, inputTokens: 10, label: 'gateway.chat' });
    expect(a.snapshot().models).toHaveLength(1);
    expect(b.snapshot().models).toEqual([]);
  });

  test('snapshot rows are copies (mutating them does not touch the ledger)', () => {
    const t = tracker();
    t.record({ modelId: HAIKU, inputTokens: 10, label: 'gateway.chat' });
    t.snapshot().models[0]!.calls = 99;
    expect(t.snapshot().models[0]!.calls).toBe(1);
  });
});

describe('reservationCostUsd', () => {
  test('free local chat model prices at 0 and never trips a cap', () => {
    expect(reservationCostUsd('ollama:llama3', 'chat', 1_000_000, 32_000)).toBe(0);
    expect(() => tracker(0.0001).reserve({ modelId: 'ollama:llama3', estimatedInputTokens: 1_000_000, maxOutputTokens: 32_000, kind: 'chat' }))
      .not.toThrow();
  });

  test('pricing override is honored; unknown price is null', () => {
    expect(reservationCostUsd('litellm:x', 'chat', 1_000_000, 1_000_000)).toBeNull();
    expect(reservationCostUsd('litellm:x', 'chat', 1_000_000, 1_000_000, { 'litellm:x': { input: 2, output: 4 } })).toBe(6);
  });

  test('parity: reserve() admits exactly when the projection fits the cap', () => {
    const projected = reservationCostUsd(HAIKU, 'chat', 10_000, 4096)!;
    expect(projected).toBeGreaterThan(0);
    const estimate = { modelId: HAIKU, estimatedInputTokens: 10_000, maxOutputTokens: 4096, kind: 'chat' as const };
    expect(() => tracker(projected * 1.01).reserve(estimate)).not.toThrow();
    expect(() => tracker(projected * 0.99).reserve(estimate)).toThrow(BudgetExhausted);
  });
});

const row = (over: Partial<ModelUsageRow>): ModelUsageRow => ({
  requested_model: HAIKU, model: HAIKU, touchpoint: 'chat', purpose: null,
  calls: 1, attempts: 1, failed_calls: 0, input_tokens: 10, output_tokens: 1, cost_usd: 0.5, cost_basis: 'measured',
  ...over,
});

describe('buildModelsUsed / mergeModelUsageRows', () => {
  test('prior-segment rows merge with the snapshot on the ledger key', () => {
    const merged = buildModelsUsed(
      { models: [row({ calls: 2, cost_usd: 1 }), row({ touchpoint: 'expansion' })] },
      [row({ calls: 3, cost_usd: 0.25, cost_basis: 'estimated' })],
    );
    expect(merged).toEqual([
      row({ calls: 5, attempts: 2, input_tokens: 20, output_tokens: 2, cost_usd: 1.25, cost_basis: 'estimated' }),
      row({ touchpoint: 'expansion' }),
    ]);
  });

  test('an unpriced contribution makes the merged cost null', () => {
    expect(mergeModelUsageRows([row({})], [row({ cost_usd: null })])[0]!.cost_usd).toBeNull();
  });
});

describe('formatModelsUsedTable', () => {
  test('header + one aligned line per row; requested -> served, attempts, failures, estimated and unpriced marks', () => {
    const lines = formatModelsUsedTable([
      row({ purpose: 'skillopt.judge', cost_usd: 0.0123 }),
      row({ touchpoint: 'expansion', requested_model: 'anthropic:haiku', attempts: 2, failed_calls: 1, cost_basis: 'estimated' }),
      row({ touchpoint: 'rerank', model: 'litellm:x', requested_model: 'litellm:x', cost_usd: null }),
    ]);
    expect(lines).toHaveLength(4);
    expect(lines[0]).toMatch(/^touchpoint\s+purpose\s+model\s+calls\s+tokens in\/out\s+cost$/);
    expect(lines[1]).toContain('skillopt.judge');
    expect(lines[1]).toContain('$0.0123');
    expect(lines[2]).toContain('engine');
    expect(lines[2]).toContain(`anthropic:haiku -> ${HAIKU}`);
    expect(lines[2]).toContain('1 (2 attempts), 1 failed');
    expect(lines[2]).toContain('~$0.5000');
    expect(lines[3]).toContain('unpriced');
    expect(lines[1]!.indexOf('10/1')).toBe(lines[3]!.indexOf('10/1'));
  });
});