Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.3 KB
Raw Blame
/**
 * #6199 — `chronicle-backfill --max-usd`: a hard spend bound fixed at queue time.
 *
 * Protects: pages queue only while queued × (per-attempt cap + one call's
 * ceiling) × maximum attempts fits under max_usd; queued rows carry the
 * campaign stamp and every executor obeys it, not current settings;
 * attempts are consumed at claim, so a crash cannot add an attempt; a stale
 * completion after its lease expired is fenced and spend is summed per
 * attempt without double counting; overlapping invocations never share a
 * page; an unpriced model under a user cap refuses with no_pricing and the
 * register-price fix, and without a cap it still warns and runs.
 * Fails when: a retry, crash, restart, settings change or second
 * invocation lets a campaign spend past its bound.
 * Seams: injected judge (`runPhaseChronicle({ judge })`) recording spend on
 * the ambient BudgetTracker; PGLite always, Postgres when DATABASE_URL is set.
 */
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../src/core/engine.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
import { operationsByName } from '../src/core/operations.ts';
import { runChronicleBackfill } from '../src/core/chronicle/backfill.ts';
import { claimChronicleRow } from '../src/core/chronicle/execute.ts';
import type { ChronicleJudge, ChronicleJudgeResult } from '../src/core/chronicle/extract-events.ts';
import type { ChronicleLedgerRow } from '../src/core/chronicle/contract.ts';
import { runPhaseChronicle } from '../src/core/cycle/chronicle.ts';
import { configureGateway, getCurrentBudgetTracker, resetGateway } from '../src/core/ai/gateway.ts';
import { recordOnTracker } from '../src/core/ai/budget-record.ts';
import { _resetBudgetTrackerWarningsForTest } from '../src/core/budget/budget-tracker.ts';
import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';

const MODEL = 'anthropic:claude-sonnet-4-6';
const UNPRICED = 'openai:gpt-unpriced-example';
const LONG = 'Alice and Bob reviewed the launch plan and agreed on the next steps for the beta release. '.repeat(2);
/** One recorded judge call: 1000 input + 100 output tokens at the sonnet rate. */
const CALL = { inputTokens: 1000, outputTokens: 100 };

type Result = Record<string, unknown> & { queued: number; worst_case_usd: number | 'unpriced'; per_page_worst_case_usd: number | null;
  fits_under_cap: number | null; cap_reached: boolean; campaign_id: string | null; next_command: string; ask_user: boolean; message: string };

function spendingJudge(answer: (n: number) => ChronicleJudgeResult | Promise<ChronicleJudgeResult>) {
  const judge = Object.assign((async () => {
    const tracker = getCurrentBudgetTracker();
    judge.caps.push(tracker?.cap);
    recordOnTracker(tracker, { modelId: MODEL, ...CALL, label: 'chronicle', kind: 'chat' });
    return answer(++judge.calls);
  }) as ChronicleJudge, { calls: 0, caps: [] as Array<number | undefined> });
  return judge;
}

for (const backend of testBackends()) {
  const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
  describe(`${backend}: chronicle-backfill --max-usd (#6199)`, () => {
    let engine: BrainEngine;
    let close: () => Promise<void>;
    const ctx = (): OperationContext => ({ engine, remote: false } as unknown as OperationContext);
    const backfill = async (p: Record<string, unknown>) => await operationsByName.chronicle_backfill.handler(ctx(), p) as Result;
    const ledger = () => engine.executeRaw<ChronicleLedgerRow>('SELECT * FROM chronicle_page_state ORDER BY slug');
    const pages = async (n: number) => {
      for (let i = 1; i <= n; i++) await engine.putPage(`meetings/m${i}`, { type: 'meeting', title: `m${i}`, compiled_truth: LONG });
    };
    const perPage = async () => (await backfill({ dry_run: true, max_usd: 1000 })).per_page_worst_case_usd!;
    const expire = () => engine.executeRaw("UPDATE chronicle_page_state SET next_attempt_at = now() - interval '1 second'");

    beforeAll(async () => { ({ engine, close } = await isolatedSharedSkillsEngine(databaseUrl)); });
    afterAll(async () => { await close(); });
    beforeEach(async () => {
      configureGateway({ chat_model: MODEL, env: {} });
      await engine.executeRaw('DELETE FROM chronicle_page_state');
      await engine.executeRaw("DELETE FROM pages WHERE slug LIKE 'meetings/%' OR slug LIKE 'life/%'");
      await engine.setConfig('auto_chronicle', 'false');
      await engine.unsetConfig('chronicle.job_budget_usd');
    });
    afterEach(() => { resetGateway(); _resetBudgetTrackerWarningsForTest(); });

    test('a dry run reports the worst case and how many pages fit; nothing is queued', async () => {
      await pages(3);
      const unit = await perPage();
      expect(unit).toBeGreaterThan(0.25 * 5);
      const r = await backfill({ dry_run: true, max_usd: unit * 2.5 });
      expect(r).toMatchObject({ queued: 2, fits_under_cap: 2, cap_reached: true, max_usd: unit * 2.5, ask_user: true });
      expect(r.worst_case_usd).toBeCloseTo(unit * 2, 3);
      expect(r.next_command).toContain(`--max-usd ${unit * 2.5} --yes`);
      expect(await ledger()).toEqual([]);
    });

    test('max_usd that fits 2 of 3 pages queues exactly 2, stamped with the campaign and its policy', async () => {
      await pages(3);
      const unit = await perPage();
      const r = await backfill({ yes: true, max_usd: unit * 2.5 });
      expect(r).toMatchObject({ queued: 2, fits_under_cap: 2, cap_reached: true, spent_usd: 0 });
      expect(r.message).toContain('did not fit under --max-usd');
      const rows = await ledger();
      expect(rows).toHaveLength(2);
      for (const row of rows) {
        expect(row).toMatchObject({ state: 'pending', trigger: 'backfill', campaign_id: r.campaign_id, max_attempts: 5, pricing_policy: 'enforced' });
        expect(Number(row.attempt_cap_usd)).toBe(0.25);
        expect(Number(row.campaign_max_usd)).toBeCloseTo(unit * 2.5, 6);
      }
    });

    test('max_usd below one page queues nothing and asks for a larger bound', async () => {
      await pages(2);
      const r = await backfill({ yes: true, max_usd: 0.5 });
      expect(r).toMatchObject({ queued: 0, fits_under_cap: 0, cap_reached: true, ask_user: true });
      expect(r.next_command).toContain('--max-usd 0.5 --dry-run');
      expect(await ledger()).toEqual([]);
    });

    test('an unpriced model under a user cap refuses with no_pricing and the register-price fix; nothing is queued', async () => {
      configureGateway({ chat_model: UNPRICED, env: {} });
      await pages(2);
      const err = await backfill({ yes: true, max_usd: 50 }).catch((e) => e);
      expect(err?.code ?? err?.error).toBe('no_pricing');
      expect(err.fix.argv.slice(0, 3)).toEqual(['gbrain', 'pricing', 'set']);
      expect(await ledger()).toEqual([]);
    });

    test('without max_usd an unpriced model warns and runs, pointing at --max-usd and price registration', async () => {
      await pages(2);
      const r = await runChronicleBackfill(engine, { yes: true, model: UNPRICED }) as unknown as Result;
      expect(r).toMatchObject({ queued: 2, worst_case_usd: 'unpriced', fits_under_cap: null, campaign_id: null });
      expect(r.message).toContain('--max-usd');
      expect(r.message).toContain('gbrain pricing set');
      expect((await ledger()).every((row) => row.campaign_id === null)).toBe(true);
    });

    test('max_usd must be a positive number', async () => {
      const err = await backfill({ dry_run: true, max_usd: -1 }).catch((e) => e);
      expect(err?.code ?? err?.error).toBe('invalid_params');
    });

    test('the executor runs a campaign row under its stamped cap after the setting changes', async () => {
      await pages(1);
      await backfill({ yes: true, max_usd: 100 });
      await engine.setConfig('chronicle.job_budget_usd', '0.05');
      const judge = spendingJudge(() => ({ events: [] }));
      await runPhaseChronicle(engine, { judge });
      expect(judge.caps).toEqual([0.25]);
      expect((await ledger())[0]).toMatchObject({ state: 'extracted', attempts: 1 });
    });

    test('a page that fails once then succeeds stays inside the bound, with spend summed per attempt', async () => {
      await pages(1);
      const unit = await perPage();
      await backfill({ yes: true, max_usd: unit + 0.01 });
      const judge = spendingJudge((n) => n === 1 ? { events: [], failure: 'chat_error' } : { events: [] });
      await runPhaseChronicle(engine, { judge });
      expect((await ledger())[0]).toMatchObject({ state: 'failed', reason: 'judge_chat_error', attempts: 1 });
      await expire();
      await runPhaseChronicle(engine, { judge });
      const [row] = await ledger();
      expect(row).toMatchObject({ state: 'extracted', attempts: 2 });
      expect(row.cost_attempts).toEqual([1, 2]);
      expect(Number(row.cost_usd)).toBeCloseTo(2 * (1000 * 3 + 100 * 15) / 1_000_000, 8);
      expect(Number(row.cost_usd)).toBeLessThanOrEqual(unit);
    });

    test('a crash after the claim consumes the attempt; an exhausted row fails without another call', async () => {
      await pages(1);
      await backfill({ yes: true, max_usd: 100 });
      for (let i = 1; i <= 5; i++) {
        const [row] = await ledger();
        const claimed = await claimChronicleRow(engine, row);
        expect(claimed?.attempts).toBe(i);
        await expire();
      }
      const [spent] = await ledger();
      expect(spent).toMatchObject({ state: 'pending', attempts: 5 });
      expect(await claimChronicleRow(engine, spent)).toBeNull();
      const judge = spendingJudge(() => ({ events: [] }));
      await runPhaseChronicle(engine, { judge });
      expect(judge.calls).toBe(0);
      expect((await ledger())[0]).toMatchObject({ state: 'failed', reason: 'campaign_exhausted', attempts: 5 });
    });

    test('a stale completion after its lease expired is fenced; its spend is recorded once', async () => {
      await pages(1);
      await backfill({ yes: true, max_usd: 100 });
      const judge = spendingJudge(async () => {
        await expire();
        const [row] = await ledger();
        expect((await claimChronicleRow(engine, row))?.attempts).toBe(2);
        return { events: [] };
      });
      await runPhaseChronicle(engine, { judge });
      const [row] = await ledger();
      expect(row).toMatchObject({ state: 'pending', attempts: 2, cost_attempts: [1] });
    });

    test('an exhausted failed revision with no lease is admitted by the preview and capped recovery', async () => {
      await pages(1);
      await backfill({ yes: true, max_usd: 100 });
      await engine.executeRaw("UPDATE chronicle_page_state SET state='failed', reason='campaign_exhausted', attempts=5, next_attempt_at=NULL");
      const [before] = await ledger();
      const unit = await perPage();
      const cap = unit + 0.01;
      expect(await backfill({ dry_run: true, max_usd: cap })).toMatchObject({ queued: 1, fits_under_cap: 1 });
      expect(await ledger()).toEqual([before]);
      expect(await backfill({ yes: true, max_usd: 0.5 })).toMatchObject({ queued: 0, cap_reached: true });
      expect(await ledger()).toEqual([before]);
      const recovered = await backfill({ yes: true, max_usd: cap });
      expect(recovered).toMatchObject({ queued: 1, fits_under_cap: 1, cap_reached: false });
      expect(recovered.worst_case_usd).toBeLessThanOrEqual(cap);
      const [after] = await ledger();
      expect(after).toMatchObject({ source_id: before.source_id, page_id: before.page_id, content_hash: before.content_hash,
        extractor_version: before.extractor_version, state: 'pending', reason: null, attempts: 0, next_attempt_at: null,
        campaign_id: recovered.campaign_id, max_attempts: 5, pricing_policy: 'enforced' });
      expect(after.campaign_id).not.toBe(before.campaign_id);
      expect(Number(after.attempt_cap_usd)).toBe(0.25);
      expect(Number(after.campaign_max_usd)).toBeCloseTo(cap, 6);
      const judge = spendingJudge(() => ({ events: [] }));
      await runPhaseChronicle(engine, { judge });
      expect(judge.calls).toBe(1);
      expect(judge.caps).toEqual([0.25]);
      expect((await ledger())[0]).toMatchObject({ state: 'extracted', attempts: 1 });
    });

    test('an exhausted failed revision keeps its active lease and recovers after expiry', async () => {
      await pages(1);
      await backfill({ yes: true, max_usd: 100 });
      await engine.executeRaw("UPDATE chronicle_page_state SET state='failed', attempts=5, next_attempt_at=now()+interval '1 hour'");
      const [leased] = await ledger();
      expect(await backfill({ yes: true, max_usd: 100 })).toMatchObject({ queued: 0 });
      expect(await ledger()).toEqual([leased]);
      await expire();
      expect(await backfill({ yes: true, max_usd: 100 })).toMatchObject({ queued: 1 });
      expect((await ledger())[0]).toMatchObject({ state: 'pending', attempts: 0, next_attempt_at: null });
    });

    test('NULL-lease recovery respects source scope and leaves other source rows intact', async () => {
      await pages(1);
      await engine.executeRaw("INSERT INTO sources (id,name) VALUES ('other-src','other-src') ON CONFLICT DO NOTHING");
      await engine.putPage('meetings/m1', { type: 'meeting', title: 'm1', compiled_truth: LONG }, { sourceId: 'other-src' });
      await backfill({ yes: true, max_usd: 100 });
      await engine.executeRaw("UPDATE chronicle_page_state SET state='pending', attempts=5, next_attempt_at=NULL");
      const before = await ledger();
      const result = await operationsByName.chronicle_backfill.handler({ ...ctx(), sourceId: 'default' }, { yes: true, max_usd: 100 }) as Result;
      expect(result.queued).toBe(1);
      const after = await ledger();
      expect(after.find((row) => row.source_id === 'other-src')).toEqual(before.find((row) => row.source_id === 'other-src'));
      expect(after.find((row) => row.source_id === 'default')).toMatchObject({ state: 'pending', attempts: 0, campaign_id: result.campaign_id });
    });

    test('a restart mid-campaign queues only the rest, each campaign within its own bound', async () => {
      await pages(4);
      const unit = await perPage();
      const first = await backfill({ yes: true, max_usd: unit * 2 + 0.01 });
      const second = await backfill({ yes: true, max_usd: unit * 2 + 0.01 });
      expect(first.queued).toBe(2);
      expect(second.queued).toBe(2);
      expect(first.campaign_id).not.toBe(second.campaign_id);
      const rows = await ledger();
      expect(rows.filter((r) => r.campaign_id === first.campaign_id)).toHaveLength(2);
      expect(rows.filter((r) => r.campaign_id === second.campaign_id)).toHaveLength(2);
      expect((await backfill({ yes: true, max_usd: unit * 2 + 0.01 })).queued).toBe(0);
    });

    test('two overlapping invocations never share a page and neither exceeds its cap', async () => {
      await pages(6);
      const unit = await perPage();
      const [a, b] = await Promise.all([backfill({ yes: true, max_usd: unit * 4 + 0.01 }), backfill({ yes: true, max_usd: unit * 3 + 0.01 })]);
      const rows = await ledger();
      expect(rows).toHaveLength(a.queued + b.queued);
      expect(a.queued * unit).toBeLessThanOrEqual(unit * 4 + 1e-9);
      expect(b.queued * unit).toBeLessThanOrEqual(unit * 3 + 1e-9);
      expect(rows.filter((r) => r.campaign_id === a.campaign_id)).toHaveLength(a.queued);
      expect(rows.filter((r) => r.campaign_id === b.campaign_id)).toHaveLength(b.queued);
    });
  });
}