Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* The per-page embedding spend cap for large pages (`embed.max_usd_per_page`,
* default $2), with its durable ledger `page_embedding_budget` (migration
* v246).
*
* Scope: pages above `pages.inline_max_bytes` (page-size-policy.ts). A page at
* or under it costs a few cents at most to embed whole and is not metered.
*
* Lifetime: spend accumulates per page for one embedding signature (model and
* dimensions); a model change starts a new ledger. The cap bounds that
* lifetime spend, so re-embed churn on a live page (boundary drift, rebuilds)
* can never spend more than the cap on it.
*
* Composition: every paid embedding entrypoint asks `planPageEmbedding` for a
* grant before it calls the provider. The grant takes the newest stale chunks
* whose estimated cost fits the page's remaining budget and reserves that
* estimate under a row lock, so concurrent sweeps never overspend. The run's
* own caps (the CLI consent cap `--max-usd`, `embed.backfill_max_usd`,
* `embed.backfill_max_usd_per_source_24h`) still meter the same provider calls
* through the gateway's budget tracker, so the lowest cap wins.
*
* States: `open`; `capped` once the cap stopped a run with stale chunks left
* (the page stays keyword-searchable and partially embedded); `held` while a
* page waits for the user's approval (the oversized-page upgrade backfill, or
* a sampled junk hit). Held and capped pages leave the stale-chunk queue
* (engine-sql/chunks.ts staleChunkWhere) so the sweep never retries them; an explicit
* approval (`gbrain embed --held --source S --max-usd N`, or `--slugs <slug>
* --source S --max-usd N`) passes `approvedCapUsd`, lifts the hold and raises the
* cap. Every printed approve and its N come from held-embedding-action.ts.
*
* Crash safety: a reservation older than an hour (the provider call never
* settled) is counted as spent, never released, so a crash cannot reopen
* budget it may already have used.
*/
import type { BrainEngine } from './engine.ts';
import { JUNK_SAMPLE_CHARS, assessJunkWindows, assessSampledJunk } from './content-sanity.ts';
import type { BodyStorage, ManifestRow } from './page-body/types.ts';
import { isSegmentedPage, pageBodyBytes } from './page-body/page-shape.ts';
import { derivationSweepCoversPage } from './page-body/segment-derive.ts';
import { estimateCostFromChars } from './embedding-pricing.ts';
import { reservationCostUsd, usageCostUsd } from './budget/reservation-cost.ts';
import { BudgetTracker, type PricingOverrides } from './budget/budget-tracker.ts';
import { getCurrentBudgetTracker, withBudgetTracker } from './ai/gateway.ts';
import { recordOnTracker } from './ai/budget-record.ts';
import { RESERVATION_WINDOW_SQL, RESERVED_AT_SQL, reserveDerivationSpend, settleLedgerReservation, type PageSpendDecision, type PageSpendGrant,
type PageSpendKind, type PageSpendSignature } from './page-spend.ts';
import { PAGE_INLINE_MAX_BYTES, budgetRowAppliesFilter, isLargePageBytes } from './page-size-policy.ts';
import { isEmbedSkipped } from './embed-skip.ts';
import { parseUsdLimit, resolveSpendPosture } from './spend-posture.ts';
import { chunkGenerationFilter } from './search/safe-chunks.ts';
import { quoteIdentifier } from './search/embedding-column.ts';
import { embeddingWriteTarget } from './page-state/projections.ts';
import { approvalAmount, approvalBasis, heldApproveAction, heldPreviewCommand, ledgerPricePerMTok, renderedCommand } from './held-embedding-action.ts';
export type { PageSpendDecision, PageSpendGrant, PageSpendKind, PageSpendSignature } from './page-spend.ts';
export { PAGE_DERIVATION_CAP_KEY } from './page-spend.ts';
export const PAGE_EMBED_CAP_KEY = 'embed.max_usd_per_page';
export const DEFAULT_PAGE_EMBED_CAP_USD = 2;
export type PageBudgetState = 'open' | 'held' | 'capped';
/** The configured cap: `embed.max_usd_per_page`, else unlimited under `spend.posture tokenmax`, else $2. `off` is Infinity. */
export async function resolvePageEmbedCapUsd(engine: Pick<BrainEngine, 'getConfig' | 'executeRaw'>): Promise<number> {
const raw = await engine.getConfig(PAGE_EMBED_CAP_KEY).catch(() => null);
if (raw === null || raw === undefined || raw === '') {
return await resolveSpendPosture(engine as BrainEngine).catch(() => 'gated') === 'tokenmax' ? Infinity : DEFAULT_PAGE_EMBED_CAP_USD;
}
return parseUsdLimit(raw, DEFAULT_PAGE_EMBED_CAP_USD);
}
export interface PageEmbedGrant {
/** chunk_index values this run may embed: the newest stale chunks that fit the remaining budget. */
chunkIndexes: ReadonlySet<number>;
reservedUsd: number;
/** Stale chunks left without a vector because the cap was reached. */
deferred: number;
capUsd: number;
/** Book the reservation as spent (`ok`) or release it (the provider call failed before billing). */
settle(outcome: { ok: boolean }): Promise<void>;
}
export type PageEmbedDecision =
| { kind: 'unmetered' }
| { kind: 'held'; reason: string }
| { kind: 'grant'; grant: PageEmbedGrant };
interface BudgetRow { state: PageBudgetState; reason: string | null; embedding_signature: string | null; spent_usd: number; reserved_usd: number; stale_reservation: boolean; cap_usd: number | null }
/** A segmented page (Wave B) carries `body_bytes` and no body strings; an inline page carries its body. */
type BudgetPage = { id: number; body_storage?: BodyStorage | null; body_bytes?: number | null; compiled_truth?: string; timeline?: string | null };
/**
* The head, middle and tail sample windows of a segmented page, read one
* segment at a time from the live manifest (never the whole body): the head
* window from the first segments, the middle window from the segment at the
* middle byte on, the tail window from the last segments back.
*/
async function segmentedSampleWindows(engine: BrainEngine, pageId: number): Promise<string[]> {
const manifest = await engine.readManifest(pageId);
if (manifest.length === 0) return [''];
const decoder = new TextDecoder('utf-8', { ignoreBOM: true });
const read = async (row: ManifestRow) => {
const texts = await engine.readSegmentTexts(row.sourceId, [row.sha256]);
return decoder.decode(texts.get(Buffer.from(row.sha256).toString('hex')) ?? new Uint8Array());
};
const forward = async (from: number) => {
let text = '';
for (let i = from; i < manifest.length && text.length < JUNK_SAMPLE_CHARS; i++) text += await read(manifest[i]!);
return text.slice(0, JUNK_SAMPLE_CHARS);
};
let tail = '';
for (let i = manifest.length - 1; i >= 0 && tail.length < JUNK_SAMPLE_CHARS; i--) tail = await read(manifest[i]!) + tail;
const totalBytes = manifest.reduce((n, row) => n + row.byteLen, 0);
const middle = manifest.findIndex(row => row.byteStart + row.byteLen > totalBytes / 2);
return [await forward(0), await forward(Math.max(0, middle)), tail.slice(-JUNK_SAMPLE_CHARS)];
}
/**
* GST-2768: holds that apply to a page of any size. A small page is otherwise unmetered, so the conversation archive's
* backfill (archive_backfill: waits for the owner's approval of the previewed cost) and its open day (archive_open_day:
* waits until the day closes or goes quiet) name the page here and the embed paths skip it until then. GST-2731: an
* imported call transcript (call_import: gbrain import --calls) is held the same way until the owner approves it.
*/
export const ANY_SIZE_HOLD_REASONS: readonly string[] = ['archive_backfill', 'archive_open_day', 'call_import'];
/**
* One point read per plan: the archive's frontmatter hold marker (transcript_archive.embedding_hold) and the page's
* budget row. An archive page names its hold in frontmatter from its first write; the first plan for it, before any
* budget row exists, records that hold, so a page published ahead of its hold row is never embedded without the
* owner's approval. An approved page keeps its row (open), so the marker holds once.
*/
async function readHolds(engine: BrainEngine, pageId: number): Promise<{ marker: string | null; state: string | null; reason: string | null }> {
const [row] = await engine.executeRaw<{ marker: string | null; state: string | null; reason: string | null }>(
`SELECT COALESCE(p.frontmatter->'transcript_archive'->>'embedding_hold', p.frontmatter->'call_import'->>'embedding_hold') AS marker, b.state, b.reason
FROM pages p LEFT JOIN page_embedding_budget b ON b.page_id = p.id WHERE p.id = $1`, [pageId]);
return row ?? { marker: null, state: null, reason: null };
}
/** Hold a large page's embedding until the user approves it (`reason` names why). Idempotent; never touches open spend. */
export async function holdPageEmbedding(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, reason: string): Promise<void> {
await engine.executeRaw(`INSERT INTO page_embedding_budget (page_id, state, reason) VALUES ($1, 'held', $2)
ON CONFLICT (page_id) DO UPDATE SET state = 'held', reason = EXCLUDED.reason, updated_at = now()`, [pageId, reason]);
}
/**
* Decide what one embedding run may spend on `page`. Small pages are
* unmetered. A held page returns `held` unless the caller carries the user's
* approval. Otherwise the newest stale chunks that fit the remaining budget are
* granted and their estimated cost is reserved.
*/
export async function planPageEmbedding(engine: BrainEngine, page: BudgetPage,
stale: ReadonlyArray<{ chunk_index: number; chunk_text: string }>,
opts: { signature?: string | null; approvedCapUsd?: number; maxMarkupRatio?: number } = {}): Promise<PageEmbedDecision> {
if (stale.length === 0) return { kind: 'unmetered' };
const approved = opts.approvedCapUsd !== undefined;
const holds = await readHolds(engine, page.id);
if (!approved && holds.state === null && holds.marker && ANY_SIZE_HOLD_REASONS.includes(holds.marker)) {
await engine.executeRaw("INSERT INTO page_embedding_budget (page_id, state, reason) VALUES ($1, 'held', $2) ON CONFLICT (page_id) DO NOTHING", [page.id, holds.marker]);
return { kind: 'held', reason: holds.marker };
}
if (!isLargePageBytes(pageBodyBytes(page))) {
if (holds.state !== 'held' || !holds.reason || !ANY_SIZE_HOLD_REASONS.includes(holds.reason)) return { kind: 'unmetered' };
if (!approved) return { kind: 'held', reason: holds.reason };
await engine.executeRaw("UPDATE page_embedding_budget SET state = 'open', reason = NULL, updated_at = now() WHERE page_id = $1", [page.id]);
return { kind: 'unmetered' };
}
const configured = await resolvePageEmbedCapUsd(engine);
const { rate, priced } = ledgerPricePerMTok(opts.signature);
return engine.transaction(async tx => {
await tx.executeRaw('INSERT INTO page_embedding_budget (page_id) VALUES ($1) ON CONFLICT (page_id) DO NOTHING', [page.id]);
const [row] = await tx.executeRaw<BudgetRow>(`SELECT state, reason, embedding_signature, spent_usd, reserved_usd, cap_usd,
(reserved_usd > 0 AND updated_at < now() - interval '1 hour') AS stale_reservation
FROM page_embedding_budget WHERE page_id = $1 FOR UPDATE`, [page.id]);
if (!row) return { kind: 'unmetered' } as const;
if (row.state === 'held' && !approved) return { kind: 'held', reason: row.reason ?? 'awaiting_approval' } as const;
if (!approved && row.state === 'open') {
const junk = isSegmentedPage(page)
? assessJunkWindows(await segmentedSampleWindows(tx, page.id), { max_markup_ratio: opts.maxMarkupRatio })
: assessSampledJunk(`${page.compiled_truth}\n${page.timeline ?? ''}`, { max_markup_ratio: opts.maxMarkupRatio });
if (junk.junk) {
const reason = junk.reasons.join(',');
await tx.executeRaw(`UPDATE page_embedding_budget SET state = 'held', reason = $2, updated_at = now() WHERE page_id = $1`, [page.id, reason]);
return { kind: 'held', reason } as const;
}
}
const sameLedger = !row.embedding_signature || !opts.signature || row.embedding_signature === opts.signature;
const spent = sameLedger ? Number(row.spent_usd) + (row.stale_reservation ? Number(row.reserved_usd) : 0) : 0;
let reserved = sameLedger && !row.stale_reservation ? Number(row.reserved_usd) : 0;
const stored = row.cap_usd === null ? configured : Number(row.cap_usd);
const capUsd = approved ? Math.max(stored, spent + reserved + opts.approvedCapUsd!) : stored;
const remaining = Math.max(0, capUsd - spent - reserved);
const chunkIndexes = new Set<number>();
let reservedUsd = 0;
for (const chunk of [...stale].sort((a, b) => b.chunk_index - a.chunk_index)) {
const cost = estimateCostFromChars(chunk.chunk_text.length, rate);
if (reservedUsd + cost > remaining) break;
reservedUsd += cost;
chunkIndexes.add(chunk.chunk_index);
}
const deferred = stale.length - chunkIndexes.size;
const openWindow = reserved > 0;
reserved += reservedUsd;
const state: PageBudgetState = deferred > 0 ? 'capped' : 'open';
const [written] = await tx.executeRaw<{ reservation_window: string | null }>(`UPDATE page_embedding_budget SET state = $2, reason = $3, embedding_signature = COALESCE($4, embedding_signature),
spent_usd = $5::float8, reserved_usd = $6::float8, ${RESERVED_AT_SQL('$6::float8', '$8::boolean')},
cap_usd = $7::float8, updated_at = now() WHERE page_id = $1 RETURNING ${RESERVATION_WINDOW_SQL} AS reservation_window`,
[page.id, state, deferred > 0 ? 'cap_reached' : null, opts.signature ?? null, spent, reserved,
approved && Number.isFinite(capUsd) ? capUsd : row.cap_usd, openWindow]);
const grant: PageEmbedGrant = {
chunkIndexes, reservedUsd, deferred, capUsd,
settle: ({ ok }) => settleLedgerReservation(engine, { table: 'page_embedding_budget', pageId: page.id }, written?.reservation_window ?? null,
reservedUsd, ok ? reservedUsd : 0, ok && !priced ? 1 : 0),
};
return { kind: 'grant', grant } as const;
});
}
/**
* Reserve `estimateUsd` of one paid step on `page` against its per-page cap (wave-b spec 8.3), before the provider call.
* An embedding reserves on page_embedding_budget under `embed.max_usd_per_page` (a new embedding model starts a new
* ledger, as planPageEmbedding does); a derivation (facts, synopsis) on its own family's ledger (page-spend.ts).
* Metered: segmented pages and pages over `pages.inline_max_bytes`; any other page is unmetered. A refusal reserves
* nothing, so the caller skips the call. A derivation runs its call through meterPaidCall, which books the actual cost.
*/
export async function reservePageSpend(engine: BrainEngine, page: BudgetPage, signature: PageSpendSignature, estimateUsd: number): Promise<PageSpendDecision> {
if (!isSegmentedPage(page) && !isLargePageBytes(pageBodyBytes(page))) return { kind: 'unmetered' };
const split = signature.indexOf(':');
const kind = signature.slice(0, split) as PageSpendKind;
if (kind !== 'embedding') return reserveDerivationSpend(engine, page.id, kind, estimateUsd);
const embeddingSignature = signature.slice(split + 1);
const configured = await resolvePageEmbedCapUsd(engine);
const estimate = Math.max(0, estimateUsd);
return engine.transaction(async tx => {
await tx.executeRaw('INSERT INTO page_embedding_budget (page_id) VALUES ($1) ON CONFLICT (page_id) DO NOTHING', [page.id]);
const [row] = await tx.executeRaw<BudgetRow>(`SELECT state, reason, embedding_signature, spent_usd, reserved_usd, cap_usd,
(reserved_usd > 0 AND updated_at < now() - interval '1 hour') AS stale_reservation
FROM page_embedding_budget WHERE page_id = $1 FOR UPDATE`, [page.id]);
if (!row) return { kind: 'unmetered' } as const;
const sameLedger = !row.embedding_signature || row.embedding_signature === embeddingSignature;
const spent = sameLedger ? Number(row.spent_usd) + (row.stale_reservation ? Number(row.reserved_usd) : 0) : 0;
const reserved = sameLedger && !row.stale_reservation ? Number(row.reserved_usd) : 0;
const capUsd = row.cap_usd === null ? configured : Number(row.cap_usd);
const refused = (reason: 'cap_reached' | 'held') => ({ kind: 'refused', reason, capUsd, spentUsd: spent, reservedUsd: reserved, estimateUsd: estimate } as const);
if (row.state === 'held') return refused('held');
if (spent + reserved + estimate > capUsd) return refused('cap_reached');
const [written] = await tx.executeRaw<{ reservation_window: string | null }>(`UPDATE page_embedding_budget SET embedding_signature = COALESCE($2, embedding_signature),
spent_usd = $3::float8, reserved_usd = $4::float8, ${RESERVED_AT_SQL('$4::float8', '$5::boolean')}, updated_at = now()
WHERE page_id = $1 RETURNING ${RESERVATION_WINDOW_SQL} AS reservation_window`,
[page.id, embeddingSignature, spent, reserved + estimate, reserved > 0]);
const grant: PageSpendGrant = {
reservedUsd: estimate, capUsd,
settle: ({ ok, usd }) => settleLedgerReservation(engine, { table: 'page_embedding_budget', pageId: page.id }, written?.reservation_window ?? null,
estimate, ok ? usd ?? estimate : 0),
};
return { kind: 'grant', grant } as const;
});
}
/** Rough characters per token and the system prompt overhead of one chat call, for a reservation estimate. */
const CHARS_PER_TOKEN = 4;
const PROMPT_OVERHEAD_TOKENS = 1500;
/** Per-1M-token rates for a chat model without a price: the reservation still bounds the spend. */
const UNPRICED_CHAT_RATE = { input: 15, output: 75 };
/**
* What one chat call over `chars` of input may cost at most (the prompt estimate plus the full output ceiling), at the
* operator's `pricing.overrides` rate when one is set: the estimate a paid derivation reserves.
*/
export function estimateChatCallUsd(model: string, chars: number, maxOutputTokens: number, pricingOverrides?: PricingOverrides): number {
const input = Math.ceil(chars / CHARS_PER_TOKEN) + PROMPT_OVERHEAD_TOKENS;
return reservationCostUsd(model, 'chat', input, maxOutputTokens, pricingOverrides)
?? (input * UNPRICED_CHAT_RATE.input + maxOutputTokens * UNPRICED_CHAT_RATE.output) / 1_000_000;
}
/** The worst case of one facts extraction (extract.ts): the call, a truncation retry at twice the output cap, and a malformed-output retry at that cap. */
export function estimateFactsExtractionUsd(model: string, chars: number, maxOutputTokens: number, pricingOverrides?: PricingOverrides): number {
return estimateChatCallUsd(model, chars, maxOutputTokens, pricingOverrides) + 2 * estimateChatCallUsd(model, chars, 2 * maxOutputTokens, pricingOverrides);
}
/**
* A metered step's tracker, which also sums what the provider may have billed: every answered attempt at its reported
* usage (a success without reported usage at its estimate), and every failed call at what failedCallUsage says its
* attempts may have billed (`billed`): each attempt, a retry wrapper's every entry included, at its reported usage
* topped up with the attempt's worst case, unless the provider definitively refused it before generating (a 4xx other
* than 408 or 499, with no usage). A 5xx, a 529, a proxy's 499 or 504, a timeout, an abort or a reset connection is
* booked. An embedding that failed without reported usage is booked as nothing (the embedding ledger's own rule).
* The step runs with the SDK's hidden retries off (providerRetries 0), so each attempt is a call it reserved for.
*
* Each attempt is also reserved and recorded on the caller's tracker (`parent`) as it happens, so the caller's cap
* (`--max-usd`) counts every in-flight attempt of every worker, its own outstanding reservations included.
*/
class MeteredStepTracker extends BudgetTracker {
billedUsd = 0;
billedAttempts = 0;
unpricedAttempts = 0;
constructor(opts: ConstructorParameters<typeof BudgetTracker>[0], private readonly parent: BudgetTracker | null, private readonly pricing?: PricingOverrides) { super(opts); }
override reserve(estimate: Parameters<BudgetTracker['reserve']>[0]): ReturnType<BudgetTracker['reserve']> {
super.reserve(estimate);
return this.parent?.reserve(estimate);
}
override release(reservation: ReturnType<BudgetTracker['reserve']>): void {
this.parent?.release(reservation);
}
override record(actual: Parameters<BudgetTracker['record']>[0]): void {
const failed = actual.failed === true || (actual.label?.endsWith('.failed') ?? false);
const tokens = failed && actual.billed ? actual.billed : { inputTokens: actual.inputTokens, outputTokens: actual.outputTokens ?? 0 };
const unbilled = failed && (((actual.kind ?? 'chat') === 'embed' && actual.estimated === true) || (tokens.inputTokens === 0 && tokens.outputTokens === 0));
if (!unbilled) {
const cost = usageCostUsd(actual.modelId, tokens.inputTokens, tokens.outputTokens, actual.kind ?? 'chat', this.pricing);
this.billedAttempts++;
if (cost === null) this.unpricedAttempts++;
else this.billedUsd += cost;
}
recordOnTracker(this.parent, actual);
super.record({ ...actual, reservation: undefined });
}
}
/**
* Run one paid derivation step on `page` under its family's cap (wave-b spec 8.3): reserve `estimateUsd` (every attempt the
* step may make), run `call`, and book what its provider calls actually cost, read from a budget tracker scoped to the call.
* Every attempt the provider answered is booked whatever the step's outcome (a truncated, malformed or refused answer, a
* throw after the provider answered), and so is every failed attempt it may have billed (MeteredStepTracker); one it
* definitively refused before generating (a rate limit, an auth error) and a failed embedding are not; an unpriced
* model books the reservation; a step with no billed attempt books nothing. The attempts are reserved and recorded on
* the caller's tracker too, so the caller's cap bounds them. `unpricedModel` names the model when the step was refused
* under an unpriced model's ceiling estimate or booked its reservation because the model has no price: the caller's
* receipt then says `unpriced_model` (unpricedModelReceipt) and doctor derivation_unpriced_model names the fix. While a fact
* purge's derivation sweep covers the page (segment-derive.ts derivationSweepCoversPage) the step is refused with
* `purge_sweep_pending` before anything is reserved or called.
*/
export async function meterPaidCall<T>(engine: BrainEngine, page: BudgetPage, signature: PageSpendSignature, estimateUsd: number,
call: () => Promise<T>, opts: { pricingOverrides?: PricingOverrides } = {}): Promise<({ refused: Extract<PageSpendDecision, { kind: 'refused' }> } | { value: T }) & { unpricedModel?: string }> {
// A purge sweep covering the page would drop the step's checkpoint, so the step is not paid for until the sweep ends.
if (await derivationSweepCoversPage(engine, page.id)) return { refused: { kind: 'refused', reason: 'purge_sweep_pending', capUsd: 0, spentUsd: 0, reservedUsd: 0, estimateUsd } };
const decision = await reservePageSpend(engine, page, signature, estimateUsd);
// PR 4 (Wave B incremental derivation, transcripts): an unpriced model reserves the ceiling estimate, so a refusal under it is the missing price's.
const model = signature.slice(signature.indexOf(':') + 1);
const unpriced = reservationCostUsd(model, 'chat', 1, 1, opts.pricingOverrides) === null ? { unpricedModel: model } : {};
if (decision.kind === 'refused') return { refused: decision, ...unpriced };
if (decision.kind === 'unmetered') return { value: await call() };
const tracker = new MeteredStepTracker({ label: `page-spend:${signature}`, pricingOverrides: opts.pricingOverrides, providerRetries: 0 }, getCurrentBudgetTracker(), opts.pricingOverrides);
try {
const value = await withBudgetTracker(tracker, call);
return { value, ...(tracker.unpricedAttempts > 0 ? { unpricedModel: model } : {}) };
} finally {
await decision.grant.settle({ ok: tracker.billedAttempts > 0, usd: tracker.unpricedAttempts > 0 ? decision.grant.reservedUsd : tracker.billedUsd });
}
}
export interface PageBudgetReport { slug: string; source_id: string; state: Exclude<PageBudgetState, 'open'>; reason: string | null;
spent_usd: number; cap_usd: number | null; pending_chunks: number; pending_chars: number }
/** Estimated provider cost of embedding `chars` of chunk text with the model in `signature` (unknown models use the conservative rate, free recipes $0). */
export function estimatePageEmbedUsd(chars: number, signature: string | null | undefined): number {
return estimateCostFromChars(chars, ledgerPricePerMTok(signature).rate);
}
/** Held and capped pages with their stale chunk counts (`gbrain embed --held`); `limit: null` lists every one. */
export async function listBudgetLimitedPages(engine: Pick<BrainEngine, 'executeRaw'>, opts: { state?: 'held' | 'capped'; sourceId?: string; limit?: number | null } = {}): Promise<PageBudgetReport[]> {
const params: unknown[] = [opts.limit === undefined ? 1000 : opts.limit];
// #7185: an embed_skip page is never embedded, so a hold on one (a backfill rewrite that did not land) is not approvable work.
const filters = [budgetRowAppliesFilter('b', 'p'), 'p.deleted_at IS NULL', `NOT (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip')`];
if (opts.state) filters.push(`b.state = $${params.push(opts.state)}`);
if (opts.sourceId) filters.push(`p.source_id = $${params.push(opts.sourceId)}`);
const pending = `(CASE WHEN p.body_storage = 'segmented' THEN cc.${quoteIdentifier((await embeddingWriteTarget(engine)).column.name)} IS NULL ELSE cc.embedded_at IS NULL END)`;
const rows = await engine.executeRaw<PageBudgetReport>(`SELECT p.slug, p.source_id, b.state, b.reason, b.spent_usd, b.cap_usd,
(SELECT count(*)::int FROM content_chunks cc WHERE cc.page_id = p.id AND ${pending} AND ${chunkGenerationFilter('p', 'cc')}) AS pending_chunks,
(SELECT COALESCE(sum(length(cc.chunk_text)), 0)::bigint FROM content_chunks cc WHERE cc.page_id = p.id AND ${pending} AND ${chunkGenerationFilter('p', 'cc')}) AS pending_chars
FROM page_embedding_budget b JOIN pages p ON p.id = b.page_id
WHERE ${filters.join(' AND ')} ORDER BY b.updated_at DESC, p.id LIMIT $1::int`, params);
return rows.map(row => ({ ...row, spent_usd: Number(row.spent_usd), cap_usd: row.cap_usd === null ? null : Number(row.cap_usd), pending_chunks: Number(row.pending_chunks), pending_chars: Number(row.pending_chars) }));
}
/** Run the provider call under a grant: booked as spent once it returns, released if it throws. */
export async function spendGrant<T>(grant: Pick<PageEmbedGrant, 'settle'> | undefined, call: () => Promise<T>): Promise<T> {
if (!grant) return call();
try {
const out = await call();
await grant.settle({ ok: true });
return out;
} catch (error) {
await grant.settle({ ok: false }).catch(() => {});
throw error;
}
}
/** The approve command for one page's waiting chunks, with a concrete --max-usd ceiling (held-embedding-action.ts). */
async function pageApproveCommand(page: { slug: string; source_id?: string }, waiting: ReadonlyArray<{ chunk_text: string }>): Promise<string> {
const basis = await approvalBasis();
const amount = approvalAmount(waiting.reduce((n, c) => n + c.chunk_text.length, 0), basis);
return renderedCommand(heldApproveAction({ source_id: page.source_id ?? 'default', slug: page.slug, pending_chunks: waiting.length, ...amount }, basis.free));
}
/**
* One embedding run's per-page step: meter `candidates` (the page's chunks
* without a vector) and narrow them to the grant, saying on stderr why chunks
* wait. Null means embed nothing now (held, or the cap left no chunk);
* otherwise `chunks` is the batch to embed, every candidate on an unmetered page.
*/
export async function meterPageBatch<C extends { chunk_index: number; chunk_text: string }>(engine: BrainEngine, page: BudgetPage & { slug: string; source_id?: string },
candidates: readonly C[], opts: { signature?: string | null; approvedCapUsd?: number; announceHeld?: boolean }): Promise<{ grant?: PageEmbedGrant; chunks: C[] } | null> {
const decision = await planPageEmbedding(engine, page, candidates, opts);
if (decision.kind === 'unmetered') return { chunks: [...candidates] };
if (decision.kind === 'held') {
if (opts.announceHeld) process.stderr.write(`[embed] ${page.slug}: embedding held (${decision.reason}); the page stays keyword-searchable. With the user's approval: ${await pageApproveCommand(page, candidates)}\n`);
return null;
}
const { grant } = decision;
const chunks = candidates.filter(c => grant.chunkIndexes.has(c.chunk_index));
if (grant.deferred) {
const waiting = candidates.filter(c => !grant.chunkIndexes.has(c.chunk_index));
process.stderr.write(`[embed] ${page.slug}: per-page cap $${grant.capUsd.toFixed(2)} reached; ${grant.deferred} older chunk(s) stay keyword-only until it rises (${await pageApproveCommand(page, waiting)})\n`);
}
if (chunks.length) return { grant, chunks };
await grant.settle({ ok: true });
return null;
}
/**
* #6722: what a write left a page's embedding in, for write, sync and
* remember receipts. Null for an ordinary page (embedded with the write, or by
* the post-write job). `deferred`: over the inline threshold or stored
* segmented, embedded after the commit under the per-page cap. `partial`: the cap stopped it with chunks
* left keyword-only. `held`: it waits for the user's approval. `skipped`: an
* embed_skip marker (`oversized` when the gate set it because the page is over
* the operator's bytes_block). Every such page is chunked and keyword-searchable.
*/
export interface PageSizeState { embedding: 'deferred' | 'partial' | 'held' | 'skipped'; reason: string; bytes: number }
export function pageSizeState(bytes: number, frontmatter: Record<string, unknown> | null | undefined,
budget?: { state: PageBudgetState; reason: string | null } | null, segmented = false): PageSizeState | null {
if (isEmbedSkipped(frontmatter)) {
return { embedding: 'skipped', reason: (frontmatter!.embed_skip as { reason?: unknown } | null)?.reason === 'oversized' ? 'oversized' : 'operator_marker', bytes };
}
const large = isLargePageBytes(bytes);
if (large && budget?.state === 'held') return { embedding: 'held', reason: budget.reason ?? 'awaiting_approval', bytes };
if (large && budget?.state === 'capped') return { embedding: 'partial', reason: 'spend_cap', bytes };
return large || segmented ? { embedding: 'deferred', reason: 'large_page', bytes } : null;
}
/**
* pageSizeState for stored pages of one source with a non-null state: the given slugs, or (null) every
* such page in the source, as a full sync walked them all.
*/
export async function readPageSizeStates(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, slugs: readonly string[] | null): Promise<Array<{ slug: string; source_id?: string } & PageSizeState>> {
if (slugs?.length === 0) return [];
const bytes = `(CASE WHEN p.body_storage = 'segmented' THEN p.body_bytes ELSE octet_length(p.compiled_truth) + octet_length(COALESCE(p.timeline, '')) END)`;
const rows = await engine.executeRaw<{ slug: string; bytes: number; segmented: boolean; frontmatter: Record<string, unknown> | null; state: PageBudgetState | null; reason: string | null }>(
`SELECT p.slug, ${bytes}::bigint AS bytes, p.body_storage = 'segmented' AS segmented, p.frontmatter, b.state, b.reason
FROM pages p LEFT JOIN page_embedding_budget b ON b.page_id = p.id
WHERE p.source_id = $1 AND p.deleted_at IS NULL${slugs ? ' AND p.slug = ANY($3::text[])' : ''}
AND (COALESCE(p.frontmatter, '{}'::jsonb) ? 'embed_skip' OR ${bytes} > $2 OR p.body_storage = 'segmented' OR b.page_id IS NOT NULL) ORDER BY p.slug`,
slugs ? [sourceId, PAGE_INLINE_MAX_BYTES, [...slugs]] : [sourceId, PAGE_INLINE_MAX_BYTES]);
return rows.flatMap(row => {
const state = pageSizeState(Number(row.bytes), row.frontmatter, row.state ? { state: row.state, reason: row.reason } : null, row.segmented === true);
return state ? [{ slug: row.slug, ...(state.embedding === 'held' ? { source_id: sourceId } : {}), ...state }] : [];
});
}
/** The receipt field for one written page: `{ size_state }` for a large or embed_skip page, else nothing (no query). */
export async function sizeStateField(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, slug: string,
page: { body_storage?: BodyStorage | null; body_bytes?: number | null; compiled_truth?: string | null; timeline?: string | null; frontmatter?: Record<string, unknown> | null } | null | undefined): Promise<{ size_state?: PageSizeState }> {
if (!page) return {};
const bytes = pageBodyBytes(page);
if (!isLargePageBytes(bytes) && !isSegmentedPage(page) && !isEmbedSkipped(page.frontmatter)) return {};
const [state] = await readPageSizeStates(engine, sourceId, [slug]);
return state ? { size_state: { embedding: state.embedding, reason: state.reason, bytes: state.bytes } } : {};
}
/** #6722: the sync receipt's tally of pages a run left large or embed_skip: the total and the first SIZE_STATE_SAMPLE. */
export interface SyncSizeStates { total: number; pages: Array<{ slug: string; path?: string; source_id?: string } & PageSizeState> }
export const SIZE_STATE_SAMPLE = 50;
export function addSizeState(tally: SyncSizeStates | undefined, entry: { slug: string; path?: string; source_id?: string } & PageSizeState): SyncSizeStates {
const pages = tally?.pages ?? [];
return { total: (tally?.total ?? 0) + 1, pages: pages.length < SIZE_STATE_SAMPLE ? [...pages, entry] : pages };
}
/** One line per sampled page for the sync output, and the rest as a count. */
export function sizeStateLines(tally: SyncSizeStates | undefined): string[] {
if (!tally?.total) return [];
const what = (s: PageSizeState & { source_id?: string }) => s.embedding === 'skipped'
? `no vectors (${s.reason === 'oversized' ? 'over the content_sanity.bytes_block limit' : 'embed_skip marker'}); chunks keep it in keyword search`
: s.embedding === 'held' ? `embedding held (${s.reason}); \`${heldPreviewCommand(s.source_id)}\` previews the cost`
: s.embedding === 'partial' ? 'partly embedded under embed.max_usd_per_page; the rest stays keyword-only'
: 'large page; embedding runs after the commit under embed.max_usd_per_page';
const lines = tally.pages.slice(0, 20).map(p => ` Size ${p.path ?? p.slug} (${p.bytes} bytes): ${what(p)}.`);
if (tally.total > lines.length) lines.push(` ... and ${tally.total - lines.length} more large or embed_skip page(s) (--json lists the first ${tally.pages.length}).`);
return lines;
}
/**
* The receipt a segmented derivation occurrence carries when its model has no price (PR 4, Wave B incremental
* derivation): an unread_windows receipt the cap stopped under the model's ceiling estimate says reason `unpriced_model`
* and the model; a finished one that booked a reservation for an unpriced call is `{ kind: 'unpriced_model', model }`.
* Doctor derivation_unpriced_model reads both. `stop` is the run's stop reason; `unread` the windows left, if any.
*/
export function unpricedModelReceipt(pipeline: string, model: string | null, stop: string | null,
unread: { windows_read: number; windows_total: number } | null): Record<string, unknown> | null {
if (unread) return { kind: 'unread_windows', pipeline, reason: model && stop === 'cap_reached' ? 'unpriced_model' : stop, ...unread, ...(model ? { model } : {}) };
return model ? { kind: 'unpriced_model', pipeline, model } : null;
}