Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from './engine.ts';
import { loadPricingOverrides } from './budget/budget-tracker.ts';
import { reservationCostUsd, usageCostUsd, type BudgetKind } from './budget/reservation-cost.ts';
import { noPricingGuidance, noPricingSteps } from './budget/no-pricing.ts';
import type { AIInvocation, AIInvocationPermit, AIInvocationUsage } from './ai/invocation-guard.ts';
import { MIGRATION_STATE_KEY, readMigrationState, type EmbeddingMigrationPlan, type MigrationState } from './embedding-migration.ts';
import type { DbLockHandle } from './db-lock.ts';
export const EMBEDDING_BUDGET_BELOW_WORST_CASE = 'embedding_budget_below_worst_case';
export const EMBEDDING_BUDGET_REFUSAL_DOCS = 'https://github.com/garrytan/gbrain/blob/master/docs/guides/write-refusals.md#embedding_budget_below_worst_case';
const USD_EPSILON = 1e-9;
/** Σ maximum input over the planned requests, from the same batcher the run dispatches through. `usd` is null only when the embedding model is unpriced. */
export interface MigrationWorstCase { requests: number; input_tokens: number; usd: number | null; unpriced_models: string[] }
export interface MigrationBudgetRefusalDetails {
error: typeof EMBEDDING_BUDGET_BELOW_WORST_CASE;
message: string;
cap_usd: number;
worst_case_usd: number;
debited_usd: number;
required_cap_usd: number;
suggestion: string;
docs: string;
}
/** Whole cents, rounded up, so a printed amount is always enough to authorize what it describes. */
export function ceilCents(usd: number): number {
return usd <= 0 ? 0 : Math.ceil(usd * 100 - USD_EPSILON) / 100;
}
export class MigrationBudgetRefusal extends Error {
readonly details: MigrationBudgetRefusalDetails;
constructor(plan: EmbeddingMigrationPlan, cap: number, worst: number, debited: number) {
const required = ceilCents(worst + debited);
const suggestion = `gbrain migrate embeddings --to ${plan.to_model} --dim ${plan.to_dims} --max-cost-usd ${required.toFixed(2)} --yes`;
const message = `Migration refused before any change (${EMBEDDING_BUDGET_BELOW_WORST_CASE}): the cap is $${cap} (--max-cost-usd), `
+ `but the worst-case authorization is $${ceilCents(worst).toFixed(2)}${debited > 0 ? ` on top of $${debited.toFixed(4)} already debited` : ''}. `
+ `No provider request was sent and no vector was dropped. Raise the cap to at least $${required.toFixed(2)}: ${suggestion}`;
super(message);
this.name = 'MigrationBudgetRefusal';
this.details = { error: EMBEDDING_BUDGET_BELOW_WORST_CASE, message, cap_usd: cap, worst_case_usd: worst, debited_usd: debited,
required_cap_usd: required, suggestion, docs: EMBEDDING_BUDGET_REFUSAL_DOCS };
}
}
/** Lease options for `assertMigrationLeases`: `rowLock: false` checks ownership without holding the lease rows (#6587). */
export interface MigrationLeaseCheck { rowLock?: boolean }
export type MigrationLeaseAssert = (tx?: BrainEngine, lease?: MigrationLeaseCheck) => Promise<void>;
/**
* The migration still owns every lease. By default the lease rows stay `FOR SHARE` until the transaction
* ends, so a steal (an ON CONFLICT update after expiry) waits for the commit; that lock also blocks the
* migration's own heartbeat refresh, so it belongs only at the end of a transaction (#6587,
* `withMigrationLeaseTransaction`). `rowLock: false` checks without holding them.
*/
export async function assertMigrationLeases(engine: BrainEngine, locks: DbLockHandle[], lease: MigrationLeaseCheck = {}) {
if (!locks.length) return;
const owned = await engine.executeRaw<{ id: string; token: string }>(`SELECT id,acquisition_token::text AS token
FROM gbrain_cycle_locks WHERE id=ANY($1::text[]) AND ttl_expires_at>now() ORDER BY id${lease.rowLock === false ? '' : ' FOR SHARE'}`, [locks.map(lock => lock.id)]);
if (owned.length !== locks.length || locks.some(lock => !owned.some(row => row.id === lock.id && row.token === lock.acquisitionToken))) {
throw new Error('Migration lease lost; authorization debit and provider dispatch refused');
}
}
const isDeadlock = (error: unknown) => typeof error === 'object' && error !== null && (error as { code?: unknown }).code === '40P01';
/**
* #6587: one migration transaction whose work (the retained-vector census, an invalidation) can run for
* minutes. The lease is checked without a row lock first, so the 15 s heartbeat can refresh it meanwhile,
* and checked again with `FOR SHARE` right before commit, so a lease lost or stolen during the work rolls
* the transaction back and a steal cannot land before the commit. `deadlockRetries` reruns a work function
* with no side effects outside the transaction when Postgres picks it as a deadlock victim (40P01) against
* concurrent writers; mutating work passes 0.
*/
export async function withMigrationLeaseTransaction<T>(engine: BrainEngine, assertOwned: MigrationLeaseAssert | undefined,
work: (tx: BrainEngine) => Promise<T>, opts: { deadlockRetries?: number } = {}): Promise<T> {
for (let attempt = 0; ; attempt++) {
try {
return await engine.transaction(async tx => {
await assertOwned?.(tx, { rowLock: false });
const result = await work(tx);
await assertOwned?.(tx);
return result;
});
} catch (error) {
if (attempt >= (opts.deadlockRetries ?? 0) || !isDeadlock(error)) throw error;
}
}
}
type MigrationBudget = NonNullable<MigrationState['budget']>;
function budgetIsValid(budget: MigrationBudget): boolean {
return Number.isFinite(budget.max_cost_usd) && budget.max_cost_usd >= 0 && Number.isFinite(budget.debited_usd)
&& budget.debited_usd >= 0 && Number.isSafeInteger(budget.requests) && budget.requests >= 0
&& (budget.pending === undefined || (typeof budget.pending === 'object' && budget.pending !== null
&& Object.values(budget.pending).every(n => Number.isFinite(n) && n >= 0)))
&& (budget.overshoot_usd === undefined || Number.isFinite(budget.overshoot_usd) && budget.overshoot_usd >= 0);
}
async function lockMigrationState(tx: BrainEngine): Promise<MigrationState | null> {
await tx.executeRaw('SELECT key FROM config WHERE key=$1 FOR UPDATE', [MIGRATION_STATE_KEY]);
return (await readMigrationState(tx)).state;
}
/**
* Durable migration authorization. Each provider attempt reserves its maximum
* input size under the migration state row lock and settles, under the same
* lock, to the provider's reported usage (#5680). A missing usage report or a
* crash before settlement keeps the maximum; usage above the reservation
* debits actual, records the overshoot and stops further dispatch. When a
* worst case is supplied, a cap below it refuses before any state changes.
*/
export async function authorizeMigrationBudget(engine: BrainEngine, plan: EmbeddingMigrationPlan, maxCostUsd?: number, locks: DbLockHandle[] = [], rerankerModel?: string, worstCase?: MigrationWorstCase) {
const prior = await readMigrationState(engine);
if (prior.corrupt) {
let legacy = false;
try {
const raw = JSON.parse(prior.raw ?? 'null');
legacy = raw !== null && typeof raw === 'object' && !Array.isArray(raw)
&& !Object.hasOwn(raw, 'budget') && !Object.hasOwn(raw, 'authorization_version');
} catch {}
if (!legacy || maxCostUsd === undefined) throw new Error('Migration state is corrupt; no paid request or invalidation was authorized. Inspect migrate embeddings --status.');
}
const same = prior.state?.to_model === plan.to_model && prior.state.to_dims === plan.to_dims;
const state: MigrationState = same ? prior.state! : {
version: 2, to_model: plan.to_model, to_dims: plan.to_dims, from_model: plan.from_model,
from_dims: plan.from_dims, started_at: new Date().toISOString(),
};
const old = state.budget;
if (state.authorization_version && !old) throw new Error('Migration authorization is missing; automatic renewal refused');
if (old && !budgetIsValid(old)) throw new Error('Migration authorization is corrupt; no request dispatched');
if (maxCostUsd !== undefined && (!Number.isFinite(maxCostUsd) || maxCostUsd < 0)) throw new Error('--max-cost-usd must be finite and nonnegative');
if (!old && maxCostUsd === undefined) throw new Error('Paid work requires explicit --max-cost-usd. Preview with --dry-run; resume retains spent authorization.');
const cap = maxCostUsd ?? old!.max_cost_usd;
const debited = old?.debited_usd ?? 0;
if (worstCase?.usd != null && cap + USD_EPSILON < debited + worstCase.usd) throw new MigrationBudgetRefusal(plan, cap, worstCase.usd, debited);
state.budget = old ? { ...old, max_cost_usd: cap } : { max_cost_usd: cap, debited_usd: 0, requests: 0 };
if (maxCostUsd !== undefined) delete state.budget.halted;
state.authorization_version = 1;
const generation = (state.authorization_generation ?? 0) + 1;
state.authorization_generation = generation;
if (!same && prior.state) {
state.retargeted_at = state.started_at;
state.superseded = [...prior.state.superseded ?? [], { to_model: prior.state.to_model, to_dims: prior.state.to_dims, started_at: prior.state.started_at }];
}
await engine.transaction(async tx => {
await assertMigrationLeases(tx, locks);
await tx.setConfig(MIGRATION_STATE_KEY, JSON.stringify(state));
});
const pricingOverrides = await loadPricingOverrides(engine);
const ownsBudget = (current: MigrationState | null): current is MigrationState & { budget: MigrationBudget } =>
!!current?.budget && current.to_model === plan.to_model && current.to_dims === plan.to_dims
&& current.authorization_generation === generation;
const settle = async (attempt: string, call: AIInvocation, kind: BudgetKind, usage: AIInvocationUsage | null) => {
await engine.transaction(async tx => {
const current = await lockMigrationState(tx);
if (!ownsBudget(current) || !budgetIsValid(current.budget)) return;
const reserved = current.budget.pending?.[attempt];
if (reserved === undefined) return;
const billable = usage && usage.inputTokens + (usage.cacheReadTokens ?? 0) + (usage.cacheWriteTokens ?? 0);
const actual = usage ? usageCostUsd(call.model, billable!, usage.outputTokens, kind, pricingOverrides) : null;
const { [attempt]: _settled, ...pending } = current.budget.pending!;
current.budget.pending = pending;
if (!Object.keys(pending).length) delete current.budget.pending;
if (actual !== null && Number.isFinite(actual) && actual >= 0) {
current.budget.debited_usd = Math.max(0, current.budget.debited_usd + actual - reserved);
if (actual > reserved + USD_EPSILON) {
current.budget.overshoot_usd = (current.budget.overshoot_usd ?? 0) + actual - reserved;
current.budget.halted = true;
}
}
await tx.setConfig(MIGRATION_STATE_KEY, JSON.stringify(current));
});
};
return async (call: AIInvocation): Promise<AIInvocationPermit> => {
if (call.kind !== 'embedding' && call.kind !== 'rerank') throw new Error('Migration permits embedding and reranker probes only');
if (call.model !== (call.kind === 'embedding' ? plan.to_model : rerankerModel)) throw new Error('Provider model differs from the authorized migration plan; no request dispatched');
if (!Number.isSafeInteger(call.maxInputTokens) || call.maxInputTokens! <= 0) throw new Error('Provider request has no conservative input ceiling; no request dispatched');
const kind: BudgetKind = call.kind === 'embedding' ? 'embed' : 'rerank';
const reserved = reservationCostUsd(call.model, kind, call.maxInputTokens!, 0, pricingOverrides);
if (reserved === null) {
throw new Error(`Migration authorization cannot price ${call.model} (${kind}); no request dispatched. ${noPricingSteps(noPricingGuidance(call.model, kind))}`);
}
const attempt = randomUUID();
await engine.transaction(async tx => {
await assertMigrationLeases(tx, locks);
const current = await lockMigrationState(tx);
if (!ownsBudget(current)) throw new Error('Migration authorization changed; no request dispatched');
const budget = current.budget;
if (!budgetIsValid(budget)) throw new Error('Migration authorization is corrupt; no request dispatched');
if (budget.halted) {
throw new Error(`Migration dispatch stopped: provider usage exceeded its reservation by $${(budget.overshoot_usd ?? 0).toFixed(6)} (recorded as overshoot). No request dispatched. Inspect gbrain migrate embeddings --status, then re-run with --max-cost-usd to renew dispatch.`);
}
if (budget.debited_usd + reserved > budget.max_cost_usd + USD_EPSILON) {
throw new Error(`Migration authorization exhausted: this request needs up to $${reserved.toFixed(6)} at its maximum input size, but $${budget.debited_usd.toFixed(6)} of the $${budget.max_cost_usd} cap (--max-cost-usd) is already debited. No request dispatched. Raise --max-cost-usd to renew; prior debits remain.`);
}
budget.debited_usd += reserved;
budget.requests++;
budget.pending = { ...budget.pending, [attempt]: reserved };
await tx.setConfig(MIGRATION_STATE_KEY, JSON.stringify(current));
});
return { settle: usage => settle(attempt, call, kind, usage) };
};
}