Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.2 KB
Raw Blame
/**
 * Durable reserve-then-settle meter for paid OAuth/MCP operations.
 *
 * Reserve-then-settle pattern (D3) prevents the "concurrent requests bust the
 * cap" race that the pre-v82 best-effort post-call recording allowed. Two
 * calls from the same OAuth client both pre-flight pass at $2 of $5,
 * both spend $2, total spend = $4 of $5 → fine. But raise the per-call
 * estimate to $3 and both calls see "$5 cap - $2 spent = $3 headroom, ok"
 * and both proceed, total spend = $8. That's the bug. The fix is atomic
 * check-and-reserve under pg_advisory_xact_lock.
 *
 * The lock key is hashed from client_id (or the group budget key). Stale reservations (worker
 * crashed before settle) become overdue after `RESERVATION_TTL_MS`.
 * Their liability remains reserved until actual usage is reconciled: a
 * timeout or UTC day change is not evidence that a provider did no work.
 *
 * Mirror of the rate-leases.ts pattern (the v0.15 rate-lease helper does
 * the same shape for outbound provider concurrency caps).
 */

import { randomUUIDv7 } from 'bun';
import type { BrainEngine } from '../engine.ts';
import { sqlQueryForEngine } from '../sql-query.ts';
import { BudgetExceededError } from '../spend-log.ts';

/** Reservation TTL — 10 minutes. Long enough for a normal provider call;
 *  an overdue hold stays charged until its outcome is known. */
export const RESERVATION_TTL_MS = 10 * 60 * 1000;

/** Generate an int hash of client_id for pg_advisory_xact_lock. */
function clientLockKey(clientId: string): number {
  // FNV-1a 32-bit hash (deterministic, no deps, fits in INT32 / BIGINT).
  let h = 0x811c9dc5;
  for (let i = 0; i < clientId.length; i++) {
    h ^= clientId.charCodeAt(i);
    h = Math.imul(h, 0x01000193);
  }
  // pg_advisory_xact_lock(BIGINT) — unsigned 32-bit value fits in BIGINT.
  return h >>> 0;
}

export interface ReserveOpts {
  clientId: string;
  estimatedCents: number;
  capCents: number | null;
  estimateKnown?: boolean;
  model: string;
  provider: string;
  jobId?: number;
  /** Trusted runtime recheck, run under the client lock before admitting IO. */
  validateAdmission?: (tx: BrainEngine) => Promise<void>;
}

export interface Reservation {
  reservationId: string;
  estimatedCents: number;
  ttlMs: number;
}

/**
 * Atomic check-and-reserve. Under `pg_advisory_xact_lock(client_id_hash)`:
 *
 *   1. Sweep expired pending reservations for this client.
 *   2. SUM today's settled spend + all unresolved reservations, across dates.
 *   3. If `committed + pending + estimated > cap`, throw `BudgetExceededError`.
 *   4. INSERT pending reservation row with TTL.
 *   5. Return reservation id.
 *
 * Lock auto-releases at transaction end (xact-scoped). All statements commit
 * or roll back as one transaction.
 */
export async function reserve(
  engine: BrainEngine,
  opts: ReserveOpts,
): Promise<Reservation> {
  assertNonEmpty('clientId', opts.clientId);
  assertFiniteNonNegative('estimatedCents', opts.estimatedCents);
  if (opts.capCents !== null) assertFiniteNonNegative('capCents', opts.capCents);
  assertNonEmpty('model', opts.model);
  assertNonEmpty('provider', opts.provider);
  if (opts.jobId !== undefined && (!Number.isSafeInteger(opts.jobId) || opts.jobId <= 0)) {
    throw new TypeError('jobId must be a positive safe integer when provided');
  }

  const reservationId = randomUUIDv7();
  const lockKey = clientLockKey(opts.clientId);
  const expiresAt = new Date(Date.now() + RESERVATION_TTL_MS);
  const todayStart = todayStartIso();

  await engine.transaction(async (tx) => {
    const sql = sqlQueryForEngine(tx);

    // Postgres can run several MCP requests for one client concurrently.
    // Hold a transaction-scoped lock across sweep + read + insert so two
    // callers cannot both observe the same headroom. PGLite serializes its
    // single connection and does not implement advisory locks.
    if (tx.kind === 'postgres') {
      // Lock-census (PR6 D5): INTENTIONALLY per-client, not per-source — the budget cap is an oauth_clients.budget_usd_per_day property; one client's concurrent requests must serialize regardless of which source each targets.
      await sql`SELECT pg_advisory_xact_lock(${BigInt(lockKey)})`;
    }
    // A cap tightened after the caller's preflight applies to this attempt.
    // The row lock also serializes this admission with in-place grant repair.
    const clients = await sql`SELECT budget_usd_per_day::text AS cap FROM oauth_clients WHERE client_id = ${opts.clientId} FOR SHARE`;
    await opts.validateAdmission?.(tx);
    const configuredCap = clients[0]?.cap == null ? null : Math.round(Number(clients[0].cap) * 100);
    const cap = opts.capCents === null ? configuredCap : configuredCap === null ? opts.capCents : Math.min(opts.capCents, configuredCap);
    if (cap !== null) assertFiniteNonNegative('configured cap', cap);

    // Step 1: sweep expired reservations for this client.
    await sql`
      UPDATE mcp_spend_reservations
         SET status = 'expired'
       WHERE client_id = ${opts.clientId}
         AND status = 'pending'
         AND expires_at < now()
    `;

    // Step 2 + 3: SUM committed + pending, refuse if over cap.
    const rows = await sql`
      SELECT
        COALESCE((
          SELECT SUM(spend_cents)::text
            FROM mcp_spend_log
           WHERE client_id = ${opts.clientId}
             AND created_at >= ${todayStart}
        ), '0') AS committed_text,
        COALESCE((
          SELECT SUM(estimated_cents)::text
            FROM mcp_spend_reservations
           WHERE client_id = ${opts.clientId}
             AND status IN ('pending', 'expired')
        ), '0') AS pending_text,
        (SELECT count(*) FROM mcp_spend_reservations
          WHERE client_id = ${opts.clientId} AND status IN ('pending','expired')
            AND estimate_known = false) AS unknown_count
    `;
    const committedCents = requiredFiniteTotal(rows[0]?.committed_text, 'committed spend');
    const pendingCents = requiredFiniteTotal(rows[0]?.pending_text, 'pending spend');
    const totalProjected = committedCents + pendingCents + opts.estimatedCents;
    if (cap !== null && (opts.estimateKnown === false || Number(rows[0]?.unknown_count ?? 0) > 0 || totalProjected > cap)) {
      throw new BudgetExceededError(
        `budget exceeded for client ${opts.clientId}: ` +
        `committed=${committedCents.toFixed(2)}¢, pending=${pendingCents.toFixed(2)}¢, ` +
        `estimated=${opts.estimatedCents.toFixed(2)}¢, cap=${cap.toFixed(2)}¢; unresolved unknown usage remains reserved`,
        committedCents + pendingCents,
        cap,
      );
    }

    // Step 4: INSERT reservation before releasing the client lock.
    await sql`
      INSERT INTO mcp_spend_reservations
        (reservation_id, client_id, job_id, estimated_cents, model, provider, status, expires_at, estimate_known, usage_unknown_reason)
      VALUES
        (${reservationId}, ${opts.clientId}, ${opts.jobId ?? null},
         ${opts.estimatedCents}, ${opts.model}, ${opts.provider}, 'pending', ${expiresAt}, ${opts.estimateKnown !== false}, ${opts.estimateKnown === false ? 'pricing_or_bound_unknown' : null})
    `;
  });

  return {
    reservationId,
    estimatedCents: opts.estimatedCents,
    ttlMs: RESERVATION_TTL_MS,
  };
}

/**
 * Why a group reservation was refused. `exhausted`: settled spend plus
 * overdue holds leave no room, so the group is done. `pressure`: only live
 * holds of concurrent attempts are in the way; a retry after they settle can
 * be admitted.
 */
export class GroupBudgetRefusal extends BudgetExceededError {
  constructor(
    readonly kind: 'exhausted' | 'pressure',
    readonly budgetKey: string,
    readonly spend: GroupSpend,
    capCents: number,
  ) {
    super(
      `budget ${kind === 'exhausted' ? 'exhausted' : 'under pressure'} for ${budgetKey}: ` +
      `committed=${spend.committedCents.toFixed(2)}¢, overdue=${spend.overdueCents.toFixed(2)}¢, ` +
      `live=${spend.liveCents.toFixed(2)}¢, cap=${capCents.toFixed(2)}¢`,
      spend.committedCents + spend.overdueCents,
      capCents,
    );
    this.name = 'GroupBudgetRefusal';
  }
}

/** A group budget's lifetime totals in cents. Overdue holds count as spent until reconciled. */
export interface GroupSpend {
  committedCents: number;
  overdueCents: number;
  liveCents: number;
}

/** Read-only group totals; a pending hold past its deadline reads as overdue. */
export async function readGroupSpend(engine: BrainEngine, budgetKey: string): Promise<GroupSpend> {
  assertNonEmpty('budgetKey', budgetKey);
  const sql = sqlQueryForEngine(engine);
  const rows = await sql`
    SELECT
      COALESCE((SELECT SUM(spend_cents)::text FROM mcp_spend_log WHERE budget_key = ${budgetKey}), '0') AS committed_text,
      COALESCE((SELECT SUM(estimated_cents)::text FROM mcp_spend_reservations
                 WHERE budget_key = ${budgetKey}
                   AND (status = 'expired' OR (status = 'pending' AND expires_at < now()))), '0') AS overdue_text,
      COALESCE((SELECT SUM(estimated_cents)::text FROM mcp_spend_reservations
                 WHERE budget_key = ${budgetKey} AND status = 'pending' AND expires_at >= now()), '0') AS live_text
  `;
  return {
    committedCents: requiredFiniteTotal(rows[0]?.committed_text, 'committed spend'),
    overdueCents: requiredFiniteTotal(rows[0]?.overdue_text, 'overdue spend'),
    liveCents: requiredFiniteTotal(rows[0]?.live_text, 'pending spend'),
  };
}

/** A reservation against a command group's shared budget (`group:<group_id>`) instead of an OAuth client. */
export type GroupReserveOpts = Omit<ReserveOpts, 'clientId' | 'validateAdmission'> & { budgetKey: string };

/**
 * Group reservation: one approved total shared by every job of a command
 * group, summed over the group's lifetime (no daily window, no client row).
 * Under the same per-key advisory lock as client reservations:
 *
 *   1. Expire this key's pending holds past their deadline (overdue).
 *   2. Settled spend plus overdue holds at or over the cap → `exhausted`.
 *   3. The attempt reserves its maximum, bounded by the remaining headroom,
 *      so one attempt can always run while nothing else is in flight; its
 *      actual cost is charged at settle, and a settled total at the cap
 *      refuses the next reservation.
 *   4. Live holds of concurrent attempts that leave no room → `pressure`.
 *
 * A null cap reserves without a check (spend is still ledgered). Unknown
 * estimates are admitted at their stated amount: the caller decides whether
 * an unpriced attempt may run.
 */
export async function reserveGroup(engine: BrainEngine, opts: GroupReserveOpts): Promise<Reservation> {
  const key = opts.budgetKey;
  assertNonEmpty('budgetKey', key);
  assertFiniteNonNegative('estimatedCents', opts.estimatedCents);
  if (opts.capCents !== null) assertFiniteNonNegative('capCents', opts.capCents);
  assertNonEmpty('model', opts.model);
  assertNonEmpty('provider', opts.provider);
  if (opts.jobId !== undefined && (!Number.isSafeInteger(opts.jobId) || opts.jobId <= 0)) {
    throw new TypeError('jobId must be a positive safe integer when provided');
  }
  const reservationId = randomUUIDv7();
  const expiresAt = new Date(Date.now() + RESERVATION_TTL_MS);
  let estimated = opts.estimatedCents;
  await engine.transaction(async (tx) => {
    const sql = sqlQueryForEngine(tx);
    if (tx.kind === 'postgres') {
      // Lock-census (PR6 D5): INTENTIONALLY per budget key — every job of a group shares one total regardless of source.
      await sql`SELECT pg_advisory_xact_lock(${BigInt(clientLockKey(key))})`;
    }
    await sql`
      UPDATE mcp_spend_reservations SET status = 'expired'
       WHERE budget_key = ${key} AND status = 'pending' AND expires_at < now()
    `;
    if (opts.capCents !== null) {
      const spend = await readGroupSpend(tx, key);
      const remaining = opts.capCents - spend.committedCents - spend.overdueCents;
      const knownFree = estimated === 0 && opts.estimateKnown !== false;
      if (remaining < 0 || remaining === 0 && !knownFree) throw new GroupBudgetRefusal('exhausted', key, spend, opts.capCents);
      estimated = Math.min(estimated, remaining);
      if (spend.liveCents > 0 && spend.liveCents + estimated > remaining) {
        throw new GroupBudgetRefusal('pressure', key, spend, opts.capCents);
      }
    }
    await sql`
      INSERT INTO mcp_spend_reservations
        (reservation_id, client_id, budget_key, job_id, estimated_cents, model, provider, status, expires_at, estimate_known, usage_unknown_reason)
      VALUES
        (${reservationId}, ${null}, ${key}, ${opts.jobId ?? null},
         ${estimated}, ${opts.model}, ${opts.provider}, 'pending', ${expiresAt}, ${opts.estimateKnown !== false}, ${opts.estimateKnown === false ? 'pricing_or_bound_unknown' : null})
    `;
  });
  return { reservationId, estimatedCents: estimated, ttlMs: RESERVATION_TTL_MS };
}

/**
 * Settle a reservation with the actual spend. Idempotent — second call
 * on the same reservation_id no-ops. Also writes a row to `mcp_spend_log`
 * so the rollup query in the next reserve sees the committed spend.
 */
export async function settle(
  engine: BrainEngine,
  reservationId: string,
  actualCents: number,
  operation: string = 'subagent_loop',
  tokenName: string | null = null,
): Promise<void> {
  assertNonEmpty('reservationId', reservationId);
  assertFiniteNonNegative('actualCents', actualCents);
  assertNonEmpty('operation', operation);

  await engine.transaction(async (tx) => {
    const sql = sqlQueryForEngine(tx);
    // A late result may arrive after the TTL sweeper marked the hold expired.
    // Settle that paid work too: truthfully recording a late overage is safer
    // than dropping it. WHERE excludes 'settled', preserving idempotency. The
    // log insert is in the same transaction, so accounting failure rolls the
    // state transition back.
    const updated = await sql`
      UPDATE mcp_spend_reservations
         SET status = 'settled',
             actual_cents = ${actualCents},
             usage_unknown_reason = NULL,
             settled_at = now()
       WHERE reservation_id = ${reservationId}
         AND status IN ('pending', 'expired')
      RETURNING client_id, budget_key, model, provider
    `;
    if (updated.length === 0) {
      const existing = await sql`
        SELECT status
          FROM mcp_spend_reservations
         WHERE reservation_id = ${reservationId}
      `;
      if (existing[0]?.status === 'settled') return;
      throw new Error(`spend reservation not found: ${reservationId}`);
    }
    const row = updated[0];
    // Mirror into mcp_spend_log so getTodaySpendCents/reserve sees it.
    await sql`
      INSERT INTO mcp_spend_log
        (client_id, budget_key, token_name, operation, spend_cents, provider, model)
      VALUES
        (${row.client_id == null ? null : String(row.client_id)}, ${row.budget_key == null ? null : String(row.budget_key)},
         ${tokenName}, ${operation}, ${actualCents},
         ${String(row.provider)}, ${String(row.model)})
    `;
  });
}

/**
 * Sweeper called by tests + the worker startup hook. Marks any
 * pending reservation past its TTL as 'expired'. This is an overdue marker,
 * not a release: unknown usage stays NULL and counts against future admission.
 *
 * Returns the number of rows expired.
 */
export async function sweepExpiredReservations(engine: BrainEngine): Promise<number> {
  const sql = sqlQueryForEngine(engine);
  const rows = await sql`
    UPDATE mcp_spend_reservations
       SET status = 'expired'
     WHERE status = 'pending'
       AND expires_at < now()
    RETURNING reservation_id
  `;
  return rows.length;
}

/** Read the per-client cap from oauth_clients.budget_usd_per_day. Returns
 *  `null` when no cap is set (legacy clients pre-v83). */
export async function getClientDailyCapCents(
  engine: BrainEngine,
  clientId: string,
): Promise<number | null> {
  assertNonEmpty('clientId', clientId);
  const sql = sqlQueryForEngine(engine);
  const rows = await sql`
    SELECT budget_usd_per_day::text AS cap
      FROM oauth_clients
     WHERE client_id = ${clientId}
  `;
  if (rows.length === 0) return null;
  const raw = rows[0]?.cap;
  if (raw === null || raw === undefined) return null;
  const usd = Number(raw);
  if (!Number.isFinite(usd) || usd < 0) {
    throw new Error(`invalid budget_usd_per_day for OAuth client ${clientId}`);
  }
  // oauth_clients stores NUMERIC(..., 2) USD, so its public cents view is
  // integral. Round to avoid binary floating-point artifacts (e.g. 0.29).
  return Math.round(usd * 100);
}

function todayStartIso(): string {
  const d = new Date();
  d.setUTCHours(0, 0, 0, 0);
  return d.toISOString();
}

function assertNonEmpty(name: string, value: string): void {
  if (typeof value !== 'string' || value.trim().length === 0) {
    throw new TypeError(`${name} must be a non-empty string`);
  }
}

function assertFiniteNonNegative(name: string, value: number): void {
  if (!Number.isFinite(value) || value < 0) {
    throw new TypeError(`${name} must be a finite non-negative number`);
  }
}

function requiredFiniteTotal(value: unknown, label: string): number {
  const total = Number(value ?? 0);
  if (!Number.isFinite(total) || total < 0) {
    throw new Error(`invalid ${label} returned by spend ledger`);
  }
  return total;
}

/** Use the lockKey helper in case future callers want it (e.g. integration tests). */
export { clientLockKey };

/** Re-export BudgetExceededError for one-stop import. */
export { BudgetExceededError };