Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix: make the SDK spend ledger a floor and derive the truth on read
Alfredo Lopeza05b2f0197ea
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex ecf62e504..d8dff2505 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.535",+ "version": "0.1.536","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.535",+ "version": "0.1.536","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex fb00c01b2..6cb79190f 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.535",+ "version": "0.1.536","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.tsindex fa31f04c5..7abf5f4e4 100644--- a/control-api/src/services/llmProviderAttemptStore.ts+++ b/control-api/src/services/llmProviderAttemptStore.ts@@ -41,6 +41,21 @@ export function isLinkedCodexUsageReady(return row.outcome === 'success' && row.usageInputTokens != null && row.usageOutputTokens != null}+/**+ * Exact token pair of a usage-ready Codex row, or null. Shared by both SDK+ * spend-floor writers (finalize and the stale-lease sweeper) so a ready Codex+ * row always freezes the same `exact` floor. Never fabricates 0/0: the null+ * re-check narrows the type without a cast and keeps a partially reported+ * usage frame out of the ledger.+ */+export function linkedCodexExactUsage(+ row: Pick<LlmProviderAttemptRow, 'outcome' | 'usageInputTokens' | 'usageOutputTokens'> | null+): { inputTokens: number; outputTokens: number } | null {+ if (!row || !isLinkedCodexUsageReady(row)) return null+ if (row.usageInputTokens == null || row.usageOutputTokens == null) return null+ return { inputTokens: row.usageInputTokens, outputTokens: row.usageOutputTokens }+}+/*** How long an authorized/redeemed Codex row may block SDK spend freeze.* After this, usage is treated as never arriving so the sweeper can close@@ -69,7 +84,16 @@ export function isLinkedCodexInFlightWithoutUsage(: row.createdAt? Date.parse(String(row.createdAt)): Number.NaN- if (Number.isFinite(created) && nowMs - created > CODEX_IN_FLIGHT_USAGE_GRACE_MS) {+ if (!Number.isFinite(created)) {+ // N-15: an unreadable created_at used to default to "still in flight",+ // which wedges the invocation behind ledger_pending forever and silently+ // hides a corrupt/absent timestamp. The grace window is undecidable here,+ // so fail loudly instead of guessing.+ throw new Error(+ 'llm_provider_attempts.created_at is missing or unparseable; the Codex in-flight grace cannot be evaluated'+ )+ }+ if (nowMs - created > CODEX_IN_FLIGHT_USAGE_GRACE_MS) {return false}return truediff --git a/control-api/src/services/pluginWorkloadSdkDb.ts b/control-api/src/services/pluginWorkloadSdkDb.tsindex 4f8d97f27..0fcf3fa38 100644--- a/control-api/src/services/pluginWorkloadSdkDb.ts+++ b/control-api/src/services/pluginWorkloadSdkDb.ts@@ -1,12 +1,18 @@import { createHash, randomUUID } from 'node:crypto'-import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'import { type DbClient, type DbTransactionClient, pool, withTransaction } from '../db.js'import type { AdministrativeEventSubmitterPrincipalV1 } from '../middleware/tracingSubmitterAuth.js'import { stableStringify } from '../utils/stableStringify.js'import {+ type LlmProviderAttemptRow,isLinkedCodexInFlightWithoutUsage,+ linkedCodexExactUsage,loadLlmProviderAttemptBySdkAttemptId,} from './llmProviderAttemptStore.js'+import {+ insertSpendOutcomeFloor,+ isOauthBrokerProvider,+ spendFloor,+} from './pluginWorkloadSdkSpendLedger.js'import {type ControlApiPermissionChange,appendControlApiPermissionEventsInTransaction,@@ -2163,13 +2169,14 @@ export async function failStaleInvocationsInTransaction([row.id, row.attempt_generation])let linkedCodexInFlight = false+ const linkedByAttempt = new Map<string, LlmProviderAttemptRow>()for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {const provider = String(attempt.provider)- if (!isLlmProviderId(provider) || PROVIDER_AUTH_MODE[provider] !== 'oauth-broker') {- continue- }+ if (!isOauthBrokerProvider(provider)) continueconst linked = await loadLlmProviderAttemptBySdkAttemptId(db, String(attempt.id))- if (linked && isLinkedCodexInFlightWithoutUsage(linked)) {+ if (!linked) continue+ linkedByAttempt.set(String(attempt.id), linked)+ if (isLinkedCodexInFlightWithoutUsage(linked)) {linkedCodexInFlight = truebreak}@@ -2192,28 +2199,32 @@ export async function failStaleInvocationsInTransaction(for (const attempt of physicalAttempts.rows as Array<Record<string, unknown>>) {// The host has disappeared, so no current JWT can prove a host_ref.- // Persist an unattributed unknown outcome instead of inventing one- // from recipe names; the physical-attempt PK makes this idempotent.- await db.query(- `INSERT INTO plugin_workload_sdk_spend_outcomes- (provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, host_ref, provider, model,- credential_slot, outcome, reason, input_tokens, output_tokens, usage_request_id)- VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9, $10, 'unknown',- 'stale_lease', NULL, NULL, NULL)- ON CONFLICT (provider_attempt_id) DO NOTHING`,- [- String(attempt.id),- row.id,- String(attempt.recipe_namespace),- String(attempt.recipe_name),- Number(attempt.attempt_generation),- Number(attempt.attempt_index),- String(attempt.target_ref),- String(attempt.provider),- String(attempt.model),- String(attempt.credential_slot),- ]+ // Persist an unattributed row instead of inventing a host identity from+ // recipe names; the physical-attempt PK makes this idempotent.+ //+ // Persist the best floor provable without a JWT: exact when the linked+ // Codex row already carries usage, unknown otherwise. Finalize derives+ // exact at read time for the in-flight residue that lands later.+ const exact = linkedCodexExactUsage(linkedByAttempt.get(String(attempt.id)) ?? null)+ await insertSpendOutcomeFloor(+ db,+ {+ providerAttemptId: String(attempt.id),+ invocationId: row.id,+ recipeNamespace: String(attempt.recipe_namespace),+ recipeName: String(attempt.recipe_name),+ attemptGeneration: Number(attempt.attempt_generation),+ attemptIndex: Number(attempt.attempt_index),+ targetRef: String(attempt.target_ref),+ hostRef: null,+ provider: String(attempt.provider),+ model: String(attempt.model),+ credentialSlot: String(attempt.credential_slot),+ reason: 'stale_lease',+ // The sweeper never ingests a usage_events row.+ usageRequestId: null,+ },+ exact ? { outcome: 'exact', ...exact } : spendFloor('unknown'))}await db.query(diff --git a/control-api/src/services/pluginWorkloadSdkFinalization.ts b/control-api/src/services/pluginWorkloadSdkFinalization.tsindex 6763952e3..0423a021e 100644--- a/control-api/src/services/pluginWorkloadSdkFinalization.ts+++ b/control-api/src/services/pluginWorkloadSdkFinalization.ts@@ -1,11 +1,19 @@-import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'import type { DbClient } from '../db.js'import {type LlmProviderAttemptRow,isLinkedCodexInFlightWithoutUsage,- isLinkedCodexUsageReady,+ linkedCodexExactUsage,loadLlmProviderAttemptBySdkAttemptId,} from './llmProviderAttemptStore.js'+import {+ type PersistableSpend,+ type SpendOutcomeRow,+ insertSpendOutcomeFloor,+ isOauthBrokerProvider,+ loadSpendOutcomeRow,+ settlePriorProviderAttemptFloors,+ spendFloor,+} from './pluginWorkloadSdkSpendLedger.js'import { withTraceIngestTransaction } from './tracing/pools.js'import { projectAcceptedUsageEvents } from './tracing/usageProjection.js'import { ingestUsageEventsInTransaction } from './usageEvents.js'@@ -83,10 +91,6 @@ function requireNonEmpty(value: string, field: string): void {}}-function isOauthBrokerProvider(provider: string): boolean {- return isLlmProviderId(provider) && PROVIDER_AUTH_MODE[provider] === 'oauth-broker'-}-function validateUsageFields(usage: PromptBridgeFinalizationUsage): void {requireNonEmpty(usage.callerRef, 'usage.callerRef')if (!Number.isInteger(usage.attemptCount) || usage.attemptCount < 1 || usage.attemptCount > 4) {@@ -159,22 +163,40 @@ function validateInput(input: PromptBridgeFinalizationInput): void {}}-function resolvePromptBridgeOutcome(+/**+ * The floor this finalization may freeze. A usage-ready Codex row always wins+ * for an oauth-broker attempt regardless of the status the host declared+ * (Addendum A.4 "best-floor writers"): the subscription was already billed and+ * the exact token pair is the strongest fact available at write time.+ */+function resolvePersistableSpend(status: PromptBridgeFinalizationStatus,oauthBroker: boolean,linked: Pick<LlmProviderAttemptRow,'status' | 'outcome' | 'usageInputTokens' | 'usageOutputTokens'- > | null-): 'exact' | 'unknown' | 'not_executed' {+ > | null,+ usage: PromptBridgeFinalizationUsage | undefined+): PersistableSpend {if (!oauthBroker) {- return status === 'complete' ? 'exact' : status === 'failed' ? 'not_executed' : 'unknown'- }- if (status === 'complete' && linked && isLinkedCodexUsageReady(linked)) {- return 'exact'+ if (status === 'complete') {+ // validateInput already guarantees this; the throw is a fail-loud guard,+ // never a fallback that would fabricate a token pair.+ if (!usage) {+ throw new PromptBridgeFinalizationError(+ 'invalid_request',+ 'complete finalization requires exact usage',+ 400+ )+ }+ return { outcome: 'exact', inputTokens: usage.inputTokens, outputTokens: usage.outputTokens }+ }+ return spendFloor(status === 'failed' ? 'not_executed' : 'unknown')}- if (status === 'failed' && !linked) return 'not_executed'- return 'unknown'+ const exact = linkedCodexExactUsage(linked)+ if (exact) return { outcome: 'exact', ...exact }+ if (status === 'failed' && !linked) return spendFloor('not_executed')+ return spendFloor('unknown')}function replayOutcomeCompatible(@@ -192,28 +214,12 @@ function replayOutcomeCompatible(Number(row.output_tokens) === input.usage?.outputTokens))}- if (input.status === 'complete') return row.outcome === 'exact' || row.outcome === 'unknown'- if (input.status === 'failed') {- return row.outcome === 'not_executed' || row.outcome === 'unknown'- }- return row.outcome === 'unknown'-}--type SpendOutcomeRow = {- provider_attempt_id: string- invocation_id: string- recipe_namespace: string- recipe_name: string- attempt_generation: number | string- attempt_index: number | string- target_ref: string- host_ref: string | null- provider: string- model: string- credential_slot: string- outcome: 'exact' | 'unknown' | 'not_executed'- input_tokens: number | string | null- output_tokens: number | string | null+ // The host is not the source of truth for oauth-broker spend (Codex is), so+ // its declared status cannot contradict an `exact` floor. Only a+ // `not_executed` floor — which proves no Codex row ever existed — can+ // contradict a host that claims the provider ran.+ if (input.status === 'failed') return true+ return row.outcome === 'exact' || row.outcome === 'unknown'}function spendBindingMatches(row: SpendOutcomeRow, input: PromptBridgeFinalizationInput): boolean {@@ -231,66 +237,91 @@ function spendBindingMatches(row: SpendOutcomeRow, input: PromptBridgeFinalizati)}-async function upgradeUnknownOauthSpendToExact(+export type SpendOutcomeView = {+ /** The immutable floor exactly as it sits in the table. */+ persisted: SpendOutcomeRow+ /** Linked Codex row, or null for a non-oauth provider / no link. */+ linked: LlmProviderAttemptRow | null+ /** Truth derived from the floor plus the linked Codex usage. */+ effective: PersistableSpend+}++/**+ * Derive the effective spend from the immutable floor plus the linked Codex+ * row. Only `unknown` is derivable: `exact` and `not_executed` are terminal+ * facts. The derivation is monotone — a floor can only ever be improved by a+ * Codex row that has already landed — so a verdict taken over the floor stays+ * stable across replays while the reported outcome catches up.+ */+export function deriveEffectiveSpend(+ persisted: Pick<SpendOutcomeRow, 'provider' | 'outcome' | 'input_tokens' | 'output_tokens'>,+ linked: Pick<LlmProviderAttemptRow, 'outcome' | 'usageInputTokens' | 'usageOutputTokens'> | null+): PersistableSpend {+ if (persisted.outcome === 'exact') {+ if (persisted.input_tokens == null || persisted.output_tokens == null) {+ // token_pair_check makes this impossible; if it happens the table is+ // corrupt and silently reporting `exact` with no tokens would undercount.+ throw new Error('exact spend outcome persisted without a token pair')+ }+ return {+ outcome: 'exact',+ inputTokens: Number(persisted.input_tokens),+ outputTokens: Number(persisted.output_tokens),+ }+ }+ if (persisted.outcome === 'unknown' && isOauthBrokerProvider(persisted.provider)) {+ const exact = linkedCodexExactUsage(linked)+ if (exact) return { outcome: 'exact', ...exact }+ }+ return spendFloor(persisted.outcome)+}++/** The single reader of the spend ledger. */+export async function loadSpendOutcome(db: DbClient,- existing: SpendOutcomeRow,- input: PromptBridgeFinalizationInput,- linked: Pick<LlmProviderAttemptRow, 'usageInputTokens' | 'usageOutputTokens'>-): Promise<PromptBridgeFinalizationResult> {- assertPersistedAuthModeCredentialRules(existing.provider, input)- if (!spendBindingMatches(existing, input)) {+ providerAttemptId: string+): Promise<SpendOutcomeView | null> {+ const persisted = await loadSpendOutcomeRow(db, providerAttemptId)+ if (!persisted) return null+ const linked = isOauthBrokerProvider(persisted.provider)+ ? await loadLlmProviderAttemptBySdkAttemptId(db, providerAttemptId)+ : null+ return { persisted, linked, effective: deriveEffectiveSpend(persisted, linked) }+}++/**+ * The only idempotent return point. Purely read-only: the floor is never+ * rewritten, so a replay can neither double-bill nor invalidate a verdict it+ * already received.+ */+function replayExistingOutcome(+ view: SpendOutcomeView,+ input: PromptBridgeFinalizationInput+): PromptBridgeFinalizationResult {+ const { persisted, linked } = view+ assertPersistedAuthModeCredentialRules(persisted.provider, input)+ // N-07: re-anchoring to the JWT's recipe binding runs BEFORE any state+ // oracle, so a foreign caller cannot learn ledger_pending/exact/unknown.+ if (!spendBindingMatches(persisted, input)) {throw new PromptBridgeFinalizationError('conflict','provider attempt was finalized with a different immutable outcome',409)}- const updated = await db.query(- `UPDATE plugin_workload_sdk_spend_outcomes- SET outcome = 'exact',- input_tokens = $2,- output_tokens = $3,- usage_request_id = COALESCE(usage_request_id, $1)- WHERE provider_attempt_id = $1- AND outcome = 'unknown'- RETURNING provider_attempt_id`,- [input.providerAttemptId, linked.usageInputTokens, linked.usageOutputTokens]- )- if ((updated.rowCount ?? updated.rows.length) !== 1) {- const raced = await db.query(- `SELECT provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, host_ref, provider,- model, credential_slot, outcome, input_tokens, output_tokens- FROM plugin_workload_sdk_spend_outcomes- WHERE provider_attempt_id = $1`,- [input.providerAttemptId]+ // Decided from the floor plus the linked Codex row, never from the+ // caller-chosen finalize status.+ if (persisted.outcome === 'unknown' && linked && isLinkedCodexInFlightWithoutUsage(linked)) {+ throw new PromptBridgeFinalizationError(+ 'ledger_pending',+ 'linked Codex attempt has not finalized usage yet',+ 409,+ true)- const existingOutcome = raced.rows[0] as SpendOutcomeRow | undefined- if (!existingOutcome) {- throw new PromptBridgeFinalizationError(- 'conflict',- 'provider attempt outcome was concurrently finalized but is not readable',- 409- )- }- return mapExistingOutcome(existingOutcome, input)- }- return {- invocationId: input.invocationId,- providerAttemptId: input.providerAttemptId,- status: input.status,- outcome: 'exact',- idempotent: false,- usageAccepted: false,}-}--function mapExistingOutcome(- row: SpendOutcomeRow,- input: PromptBridgeFinalizationInput-): PromptBridgeFinalizationResult {- assertPersistedAuthModeCredentialRules(row.provider, input)- if (!spendBindingMatches(row, input) || !replayOutcomeCompatible(input, row)) {+ // N-05: judged against the immutable floor, so the verdict cannot change+ // between two replays of the same request.+ if (!replayOutcomeCompatible(input, persisted)) {throw new PromptBridgeFinalizationError('conflict','provider attempt was finalized with a different immutable outcome',@@ -301,18 +332,24 @@ function mapExistingOutcome(invocationId: input.invocationId,providerAttemptId: input.providerAttemptId,status: input.status,- outcome: row.outcome,+ outcome: view.effective.outcome,idempotent: true,- usageAccepted: row.outcome === 'exact' && !isOauthBrokerProvider(row.provider),+ usageAccepted: persisted.outcome === 'exact' && !isOauthBrokerProvider(persisted.provider),}}/*** Finalize one physical promptBridge attempt and its logical invocation in a* single trace-ingest transaction. Exact usage is inserted through the same- * receipt binder as the normal usage endpoint; unknown provider spend and- * proven no-execution failures get immutable ledger rows instead of- * disappearing behind a public provider_unavailable response.+ * receipt binder as the normal usage endpoint.+ *+ * The spend row written here is an immutable FLOOR — the least spend provable+ * at write time — and is never updated. For an oauth-broker attempt the+ * effective outcome is derived at read time from the linked+ * `llm_provider_attempts` usage, so an `unknown` floor whose Codex usage lands+ * later reports `exact` without any write. A `failed`/`provider_unavailable`+ * finalization can therefore legitimately report `exact`: the host's own+ * account of the attempt is preserved in the row's `reason`.*/export async function finalizePromptBridge(input: PromptBridgeFinalizationInput@@ -330,51 +367,8 @@ export async function finalizePromptBridgeInTransaction(const recipeLock = `plugin_workload_sdk:${input.recipeNamespace}/${input.recipeName}`await db.query(`SELECT pg_advisory_xact_lock(hashtext($1)::bigint)`, [recipeLock])- const existingResult = await db.query(- `SELECT provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, provider, model,- credential_slot, host_ref, outcome, input_tokens, output_tokens- FROM plugin_workload_sdk_spend_outcomes- WHERE provider_attempt_id = $1`,- [input.providerAttemptId]- )- const existing = existingResult.rows[0] as SpendOutcomeRow | undefined- if (existing) {- if (isOauthBrokerProvider(existing.provider)) {- const linkedCodex = await loadLlmProviderAttemptBySdkAttemptId(db, input.providerAttemptId)- // Upgrade and ledger_pending are decided from persisted spend + the- // linked Codex row, not from the caller-chosen finalize status.- if (- existing.outcome === 'unknown' &&- linkedCodex &&- isLinkedCodexInFlightWithoutUsage(linkedCodex)- ) {- throw new PromptBridgeFinalizationError(- 'ledger_pending',- 'linked Codex attempt has not finalized usage yet',- 409,- true- )- }- if (existing.outcome === 'unknown' && linkedCodex && isLinkedCodexUsageReady(linkedCodex)) {- return upgradeUnknownOauthSpendToExact(db, existing, input, linkedCodex)- }- if (- existing.outcome === 'exact' &&- linkedCodex &&- isLinkedCodexUsageReady(linkedCodex) &&- (Number(existing.input_tokens) !== Number(linkedCodex.usageInputTokens) ||- Number(existing.output_tokens) !== Number(linkedCodex.usageOutputTokens))- ) {- throw new PromptBridgeFinalizationError(- 'conflict',- 'provider attempt was finalized with a different immutable outcome',- 409- )- }- }- return mapExistingOutcome(existing, input)- }+ const existing = await loadSpendOutcome(db, input.providerAttemptId)+ if (existing) return replayExistingOutcome(existing, input)const invocationResult = await db.query(`SELECT id, recipe_namespace, recipe_name, method, status, attempt_generation@@ -446,11 +440,10 @@ export async function finalizePromptBridgeInTransaction(}assertPersistedAuthModeCredentialRules(String(providerAttempt.provider), input)- const expectedStatus = input.statusconst currentInvocationStatus = String(invocation.status)const currentReceiptStatus = String(receipt.status)const currentProviderStatus = String(providerAttempt.status)- const compatibleStatuses = new Set(['in_progress', expectedStatus])+ const compatibleStatuses = new Set(['in_progress', input.status])// SDK-only success is finalized atomically by this service, so the normal// path leaves the physical attempt in_progress until usage/outcome commit.// Accept terminal provider states on the unknown path as a recovery fence:@@ -490,7 +483,20 @@ export async function finalizePromptBridgeInTransaction()}}- const outcome = resolvePromptBridgeOutcome(input.status, oauthBroker, linkedCodex)+ const spend = resolvePersistableSpend(input.status, oauthBroker, linkedCodex, input.usage)+ // N-17: only point a usage_request_id at a usage_events row that is really+ // written. Codex spend is ingested by the proxy finalize, not here.+ const willIngestUsage = input.status === 'complete' && !oauthBroker+ // Codex already billed this attempt, so closing the invocation `failed`+ // would leave it revivable (reviveFailedInvocation gates only on+ // status = 'failed') and the same idempotency key could launch a second+ // billable Codex call. provider_unavailable is the non-revivable terminal+ // state for exactly this situation. The RESULT still echoes input.status:+ // mcp-host's controlApiClient requires result.status === body.status.+ const expectedStatus: PromptBridgeFinalizationStatus =+ oauthBroker && spend.outcome === 'exact' && input.status === 'failed'+ ? 'provider_unavailable'+ : input.statuslet usageAccepted = false// Promote the immutable physical and attempt receipts before exact usage is@@ -503,7 +509,7 @@ export async function finalizePromptBridgeInTransaction(lease_expires_at = NULL,usage_request_id = CASE WHEN $3 THEN $1 ELSE usage_request_id ENDWHERE id = $1`,- [input.providerAttemptId, expectedStatus, input.status === 'complete']+ [input.providerAttemptId, expectedStatus, willIngestUsage])await db.query(`UPDATE plugin_workload_sdk_invocation_attempts@@ -512,7 +518,7 @@ export async function finalizePromptBridgeInTransaction([input.invocationId, expectedStatus, input.attemptGeneration])- if (input.status === 'complete' && !oauthBroker) {+ if (willIngestUsage) {const usage = input.usage!const event = {request_id: input.providerAttemptId,@@ -579,65 +585,61 @@ export async function finalizePromptBridgeInTransaction(WHERE id = $1 AND attempt_generation = $3`,[input.invocationId, expectedStatus, input.attemptGeneration])- const spendOutcomeInsert = await db.query(- `INSERT INTO plugin_workload_sdk_spend_outcomes- (provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, host_ref, provider, model,- credential_slot, outcome, reason, input_tokens, output_tokens, usage_request_id)- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)- ON CONFLICT (provider_attempt_id) DO NOTHING- RETURNING provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, host_ref, provider,- model, credential_slot, outcome, input_tokens, output_tokens`,- [- input.providerAttemptId,- input.invocationId,- input.recipeNamespace,- input.recipeName,- input.attemptGeneration,- input.providerAttemptIndex,- input.target.targetRef,- input.hostRef,- input.target.provider,- input.target.model,- input.target.credentialSlot,- outcome,- input.reason,- oauthBroker ? (linkedCodex?.usageInputTokens ?? null) : (input.usage?.inputTokens ?? null),- oauthBroker ? (linkedCodex?.usageOutputTokens ?? null) : (input.usage?.outputTokens ?? null),- input.status === 'complete' ? input.providerAttemptId : null,- ]+ const insertedFloor = await insertSpendOutcomeFloor(+ db,+ {+ providerAttemptId: input.providerAttemptId,+ invocationId: input.invocationId,+ recipeNamespace: input.recipeNamespace,+ recipeName: input.recipeName,+ attemptGeneration: input.attemptGeneration,+ attemptIndex: input.providerAttemptIndex,+ targetRef: input.target.targetRef,+ hostRef: input.hostRef,+ provider: input.target.provider,+ model: input.target.model,+ credentialSlot: input.target.credentialSlot,+ reason: input.reason,+ usageRequestId: willIngestUsage ? input.providerAttemptId : null,+ },+ spend)+ // RP-539-003: a successful failover finalizes only its winner, so the+ // attempts it displaced would never reach this endpoint. They are terminal+ // and immutable by the reservation fence, so they can be discovered and+ // settled here without trusting the host to declare them. Index 1 has no+ // predecessors, so the query is skipped rather than run over an empty range.+ if (input.providerAttemptIndex > 1) {+ await settlePriorProviderAttemptFloors(db, {+ invocationId: input.invocationId,+ attemptGeneration: input.attemptGeneration,+ attemptIndex: input.providerAttemptIndex,+ hostRef: input.hostRef,+ })+ }+// The stale-lease sweeper can win the physical-attempt race between our// initial ledger lookup and this insert. Treat an identical existing outcome// as an idempotent replay and surface a stable conflict for incompatible// accounting, never a raw unique_violation/500.- if ((spendOutcomeInsert.rowCount ?? spendOutcomeInsert.rows.length) === 0) {- const raced = await db.query(- `SELECT provider_attempt_id, invocation_id, recipe_namespace, recipe_name,- attempt_generation, attempt_index, target_ref, host_ref, provider,- model, credential_slot, outcome, input_tokens, output_tokens- FROM plugin_workload_sdk_spend_outcomes- WHERE provider_attempt_id = $1`,- [input.providerAttemptId]- )- const existingOutcome = raced.rows[0] as SpendOutcomeRow | undefined- if (!existingOutcome) {+ if (!insertedFloor) {+ const raced = await loadSpendOutcome(db, input.providerAttemptId)+ if (!raced) {throw new PromptBridgeFinalizationError('conflict','provider attempt outcome was concurrently finalized but is not readable',409)}- return mapExistingOutcome(existingOutcome, input)+ return replayExistingOutcome(raced, input)}return {invocationId: input.invocationId,providerAttemptId: input.providerAttemptId,status: input.status,- outcome,+ outcome: deriveEffectiveSpend(insertedFloor, linkedCodex).outcome,idempotent: false,usageAccepted,}diff --git a/control-api/src/services/pluginWorkloadSdkSchema.ts b/control-api/src/services/pluginWorkloadSdkSchema.tsindex 600ad48ca..ef62f61b7 100644--- a/control-api/src/services/pluginWorkloadSdkSchema.ts+++ b/control-api/src/services/pluginWorkloadSdkSchema.ts@@ -574,6 +574,14 @@ export async function addPluginWorkloadSdkCredentialTicketRuntimeAccess(* executed (for example, a timeout or a lost completion acknowledgement).* Rows are immutable and keyed by the physical attempt id so retries of the* finalization request are safe.+ *+ * Rows are the immutable FLOOR: the least spend provable at write time. For+ * oauth-broker attempts the effective outcome is derived at read time from the+ * linked `llm_provider_attempts` usage (`loadSpendOutcome`); an `unknown` row+ * is never rewritten to `exact`. Any raw SQL that reads this table without+ * that join therefore reads a lower bound, not the effective spend — the+ * billing truth for Codex already lives in `usage_events`, written by the+ * proxy finalize.*/export async function addPluginWorkloadSdkSpendOutcomeLedger(db: DbClient): Promise<void> {await db.query(`diff --git a/control-api/src/services/pluginWorkloadSdkSpendLedger.ts b/control-api/src/services/pluginWorkloadSdkSpendLedger.tsnew file mode 100644index 000000000..8526d8fd7--- /dev/null+++ b/control-api/src/services/pluginWorkloadSdkSpendLedger.ts@@ -0,0 +1,212 @@+import { PROVIDER_AUTH_MODE, isLlmProviderId } from '@clerum/llm-providers'+import type { DbClient } from '../db.js'+import {+ linkedCodexExactUsage,+ loadLlmProviderAttemptBySdkAttemptId,+} from './llmProviderAttemptStore.js'++// ─── Plugin Workload SDK — immutable spend floor ─────────────────────────+// `plugin_workload_sdk_spend_outcomes` rows are an append-only FLOOR: the+// least spend provable at write time. They are never updated (the runtime role+// holds SELECT/INSERT only). The effective truth of an oauth-broker attempt is+// derived at read time from the linked `llm_provider_attempts` usage — see+// `deriveEffectiveSpend` in pluginWorkloadSdkFinalization.ts.+//+// This module owns the ONLY INSERT statement against the table so that both+// writers (finalize and the stale-lease sweeper) share one shape and+// `token_pair_check` is guaranteed by the `PersistableSpend` type rather than+// by tests.++export type SpendOutcomeRow = {+ provider_attempt_id: string+ invocation_id: string+ recipe_namespace: string+ recipe_name: string+ attempt_generation: number | string+ attempt_index: number | string+ target_ref: string+ host_ref: string | null+ provider: string+ model: string+ credential_slot: string+ outcome: 'exact' | 'unknown' | 'not_executed'+ input_tokens: number | string | null+ output_tokens: number | string | null+}++/**+ * A spend value that can be persisted. The union makes the database's+ * `token_pair_check` unrepresentable to violate: `exact` always carries both+ * token counts, `unknown`/`not_executed` never carry either.+ */+export type PersistableSpend =+ | { outcome: 'exact'; inputTokens: number; outputTokens: number }+ | { outcome: 'unknown' | 'not_executed'; inputTokens: null; outputTokens: null }++export function spendFloor(outcome: 'unknown' | 'not_executed'): PersistableSpend {+ return { outcome, inputTokens: null, outputTokens: null }+}++export function isOauthBrokerProvider(provider: string): boolean {+ return isLlmProviderId(provider) && PROVIDER_AUTH_MODE[provider] === 'oauth-broker'+}++export const SPEND_OUTCOME_COLUMNS = `provider_attempt_id, invocation_id, recipe_namespace, recipe_name,+ attempt_generation, attempt_index, target_ref, host_ref, provider,+ model, credential_slot, outcome, input_tokens, output_tokens`++const SPEND_OUTCOME_SELECT = `SELECT ${SPEND_OUTCOME_COLUMNS}+ FROM plugin_workload_sdk_spend_outcomes+ WHERE provider_attempt_id = $1`++export async function loadSpendOutcomeRow(+ db: DbClient,+ providerAttemptId: string+): Promise<SpendOutcomeRow | null> {+ const result = await db.query(SPEND_OUTCOME_SELECT, [providerAttemptId])+ return (result.rows[0] as SpendOutcomeRow | undefined) ?? null+}++export type SpendOutcomeFloorInput = {+ providerAttemptId: string+ invocationId: string+ recipeNamespace: string+ recipeName: string+ attemptGeneration: number+ attemptIndex: number+ targetRef: string+ /** NULL when no runtime JWT proves the host (stale-lease recovery). */+ hostRef: string | null+ provider: string+ model: string+ credentialSlot: string+ reason: string+ /** Set only when this finalization actually ingested a `usage_events` row. */+ usageRequestId: string | null+}++/**+ * The single INSERT into the spend ledger. `ON CONFLICT DO NOTHING` on the+ * physical-attempt primary key makes every writer idempotent against the+ * others; a null return means another writer already froze this attempt's+ * floor and the caller must re-read it.+ */+export async function insertSpendOutcomeFloor(+ db: DbClient,+ row: SpendOutcomeFloorInput,+ spend: PersistableSpend+): Promise<SpendOutcomeRow | null> {+ const inserted = await db.query(+ `INSERT INTO plugin_workload_sdk_spend_outcomes+ (provider_attempt_id, invocation_id, recipe_namespace, recipe_name,+ attempt_generation, attempt_index, target_ref, host_ref, provider, model,+ credential_slot, outcome, reason, input_tokens, output_tokens, usage_request_id)+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)+ ON CONFLICT (provider_attempt_id) DO NOTHING+ RETURNING ${SPEND_OUTCOME_COLUMNS}`,+ [+ row.providerAttemptId,+ row.invocationId,+ row.recipeNamespace,+ row.recipeName,+ row.attemptGeneration,+ row.attemptIndex,+ row.targetRef,+ row.hostRef,+ row.provider,+ row.model,+ row.credentialSlot,+ spend.outcome,+ row.reason,+ spend.inputTokens,+ spend.outputTokens,+ row.usageRequestId,+ ]+ )+ if ((inserted.rowCount ?? inserted.rows.length) === 0) return null+ return inserted.rows[0] as SpendOutcomeRow+}++const TERMINAL_PRIOR_ATTEMPT_STATUSES = new Set(['skipped', 'failed', 'provider_unavailable'])++/**+ * Best floor provable for a physical attempt that was already closed before+ * the one being finalized. A ready Codex row wins regardless of the physical+ * status (the subscription was billed), otherwise the status decides.+ */+async function resolvePriorAttemptSpend(+ db: DbClient,+ attemptId: string,+ provider: string,+ status: string+): Promise<PersistableSpend> {+ if (!TERMINAL_PRIOR_ATTEMPT_STATUSES.has(status)) {+ // reservePluginWorkloadSdkProviderAttempt refuses to hand out index k+1+ // unless attempt k is already terminal, and no writer moves a terminal+ // attempt back. Reaching this means that fence broke: fail loudly rather+ // than freeze a floor over a still-running provider call.+ throw new Error(+ `prior provider attempt ${attemptId} is not terminal (status=${status}); the reservation fence is broken`+ )+ }+ if (isOauthBrokerProvider(provider)) {+ const exact = linkedCodexExactUsage(await loadLlmProviderAttemptBySdkAttemptId(db, attemptId))+ if (exact) return { outcome: 'exact', ...exact }+ }+ return spendFloor(status === 'provider_unavailable' ? 'unknown' : 'not_executed')+}++/**+ * RP-539-003: on a successful failover only the winning attempt reaches+ * `finalize`, so every attempt it displaced would stay terminal without a+ * spend row — the legacy status endpoint does not touch the ledger and the+ * sweeper only visits `in_progress` invocations. Discover those attempts from+ * the database instead of trusting the host to declare them, and freeze one+ * floor each. Invocation and receipt rows are deliberately untouched: the+ * winning attempt already owns their transition.+ */+export async function settlePriorProviderAttemptFloors(+ db: DbClient,+ scope: {+ invocationId: string+ attemptGeneration: number+ attemptIndex: number+ hostRef: string+ }+): Promise<void> {+ const priors = await db.query(+ `SELECT id, recipe_namespace, recipe_name, attempt_index, target_ref,+ provider, model, credential_slot, status+ FROM plugin_workload_sdk_provider_attempts+ WHERE invocation_id = $1+ AND attempt_generation = $2+ AND attempt_index < $3+ ORDER BY attempt_index+ FOR UPDATE`,+ [scope.invocationId, scope.attemptGeneration, scope.attemptIndex]+ )+ for (const prior of priors.rows as Array<Record<string, unknown>>) {+ const attemptId = String(prior.id)+ const provider = String(prior.provider)+ const spend = await resolvePriorAttemptSpend(db, attemptId, provider, String(prior.status))+ await insertSpendOutcomeFloor(+ db,+ {+ providerAttemptId: attemptId,+ invocationId: scope.invocationId,+ recipeNamespace: String(prior.recipe_namespace),+ recipeName: String(prior.recipe_name),+ attemptGeneration: scope.attemptGeneration,+ attemptIndex: Number(prior.attempt_index),+ targetRef: String(prior.target_ref),+ hostRef: scope.hostRef,+ provider,+ model: String(prior.model),+ credentialSlot: String(prior.credential_slot),+ reason: `prior_attempt_${String(prior.status)}`,+ usageRequestId: null,+ },+ spend+ )+ }+}diff --git a/control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts b/control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.tsindex ba52bdb1c..f566015ac 100644--- a/control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts+++ b/control-api/test/pluginWorkloadSdkCodexDualLedger.realPostgres.integration.test.ts@@ -548,6 +548,50 @@ describeRealPostgres('Plugin Workload SDK Codex dual ledger on real PostgreSQL',output_tokens: 7,credential_slot: '',})++ // Addendum A.4.3: the host is not the source of truth for oauth-broker+ // spend, so a later `failed` report cannot contradict an exact floor that+ // Codex itself proved. It used to 409, which made the host's retry of a+ // lost response look like an accounting conflict.+ await expect(+ finalizePromptBridgeInTransaction(+ {+ invocationId,+ recipeNamespace: NS,+ recipeName: RECIPE,+ hostRef: `${NS}/${RECIPE}`,+ attemptGeneration: 1,+ providerAttemptId: sdkAttemptId,+ providerAttemptIndex: 1,+ status: 'failed',+ target: {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: MODEL,+ credentialSlot: '',+ },+ reason: 'provider_stream_failed',+ },+ pool+ )+ ).resolves.toMatchObject({+ status: 'failed',+ outcome: 'exact',+ idempotent: true,+ usageAccepted: false,+ })+ const unchanged = await pool.query(+ `SELECT outcome, input_tokens, output_tokens, reason+ FROM plugin_workload_sdk_spend_outcomes+ WHERE provider_attempt_id = $1`,+ [sdkAttemptId]+ )+ expect(unchanged.rows[0]).toEqual({+ outcome: 'exact',+ input_tokens: 12,+ output_tokens: 7,+ reason: 'provider_completed',+ })})it('finalizes a pre-dispatch Codex miss as not_executed', async () => {diff --git a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts b/control-api/test/services.llmProviderAttemptStore.helpers.test.tsindex c80cde6b8..4a19cc242 100644--- a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts+++ b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts@@ -3,6 +3,7 @@ import {CODEX_IN_FLIGHT_USAGE_GRACE_MS,isLinkedCodexInFlightWithoutUsage,isLinkedCodexUsageReady,+ linkedCodexExactUsage,} from '../src/services/llmProviderAttemptStore.js'const recentCreatedAt = new Date('2026-09-02T07:40:00.000Z')@@ -126,4 +127,48 @@ describe('linked Codex usage predicates', () => {)).toBe(false)})++ it('fails loudly instead of pinning an attempt in flight forever when created_at is unreadable', () => {+ // N-15: the old default returned true, which wedges the invocation behind+ // ledger_pending for good and hides the corrupt timestamp.+ for (const createdAt of [null, undefined, 'not-a-timestamp']) {+ expect(() =>+ isLinkedCodexInFlightWithoutUsage(+ {+ status: 'authorized',+ outcome: null,+ usageInputTokens: null,+ usageOutputTokens: null,+ createdAt,+ },+ nowMs+ )+ ).toThrow(/created_at is missing or unparseable/)+ }+ })+})++describe('linkedCodexExactUsage', () => {+ it('returns the token pair only for a usage-ready row', () => {+ expect(+ linkedCodexExactUsage({ outcome: 'success', usageInputTokens: 12, usageOutputTokens: 7 })+ ).toEqual({ inputTokens: 12, outputTokens: 7 })+ // A genuine zero-token success is not the same as a missing pair.+ expect(+ linkedCodexExactUsage({ outcome: 'success', usageInputTokens: 0, usageOutputTokens: 0 })+ ).toEqual({ inputTokens: 0, outputTokens: 0 })+ })++ it('never fabricates a token pair', () => {+ expect(linkedCodexExactUsage(null)).toBeNull()+ expect(+ linkedCodexExactUsage({ outcome: 'error', usageInputTokens: 12, usageOutputTokens: 7 })+ ).toBeNull()+ expect(+ linkedCodexExactUsage({ outcome: 'success', usageInputTokens: 12, usageOutputTokens: null })+ ).toBeNull()+ expect(+ linkedCodexExactUsage({ outcome: 'success', usageInputTokens: null, usageOutputTokens: null })+ ).toBeNull()+ })})diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex a131cc4e3..4d1a5d62c 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -10,10 +10,12 @@ import {hasUsableClientNotificationRecipients,hashPromptTargetPolicy,markPluginWorkloadSdkProviderAttemptStatus,+ promoteReservedOauthBrokerProviderAttempt,redeemPluginWorkloadSdkCredentialTicketJti,registerPluginWorkloadSdkCredentialTicketJti,reservePluginWorkloadSdkProviderAttempt,resolveRecipientProfiles,+ reviveFailedInvocation,revokePluginWorkloadSdkForRecipe,updateInvocationStatus,upsertGrant,@@ -588,13 +590,135 @@ describe('invocation retry lifecycle timestamps', () => {} as never).mockResolvedValue({ rows: [{ id: 'inv-codex-stale' }], rowCount: 1 } as never)await expect(failStaleInvocations(150)).resolves.toBe(1)- expect(- vi- .mocked(pool.query)- .mock.calls.some(([sql]: [string]) =>- sql.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')- )- ).toBe(true)+ const insert = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]: [string]) =>+ sql.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ ) as unknown as [string, unknown[]] | undefined+ expect(insert).toBeDefined()+ // No usage ever arrived, so unknown is the honest floor. Finalize derives+ // exact later if the Codex row lands.+ expect(insert?.[1]?.slice(11, 16)).toEqual(['unknown', 'stale_lease', null, null, null])+ })++ it('freezes oauth spend as exact when the linked Codex usage is already ready', async () => {+ vi.mocked(pool.query)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'inv-codex-ready',+ attempt_generation: 1,+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ },+ ],+ rowCount: 1,+ } as never)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'attempt-codex-ready',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ attempt_generation: 1,+ attempt_index: 1,+ target_ref: 'primary-codex',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credential_slot: '',+ },+ ],+ rowCount: 1,+ } as never)+ .mockResolvedValueOnce({+ rows: [+ {+ id: 'codex-ready',+ caller_kind: 'recipe',+ host_ref: 'sandbox-recipes/sdk-recipe',+ recipe_namespace: 'sandbox-recipes',+ recipe_name: 'sdk-recipe',+ invocation_id: 'inv-codex-ready',+ attempt_generation: 1,+ provider_attempt_index: 1,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ request_hash: 'hash',+ policy_revision: 1,+ policy_hash: 'a'.repeat(64),+ budget_reservation_id: 'budget-1',+ connection_revision: 1,+ plugin_workload_sdk_provider_attempt_id: 'attempt-codex-ready',+ status: 'finalized',+ outcome: 'success',+ usage_input_tokens: 12,+ usage_output_tokens: 7,+ created_at: new Date(),+ },+ ],+ rowCount: 1,+ } as never)+ .mockResolvedValue({ rows: [{ id: 'inv-codex-ready' }], rowCount: 1 } as never)++ await expect(failStaleInvocations(150)).resolves.toBe(1)++ // Addendum A.4 "best-floor writers": a ready Codex row freezes exact in+ // BOTH writers, so the sweeper no longer under-reports spend it can prove+ // without a JWT. host_ref stays NULL (no host to attribute) and+ // usage_request_id stays NULL (the sweeper ingests no usage_events row).+ const insert = vi+ .mocked(pool.query)+ .mock.calls.find(([sql]: [string]) =>+ sql.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ ) as unknown as [string, unknown[]] | undefined+ expect(insert?.[1]?.[7]).toBeNull()+ expect(insert?.[1]?.slice(11, 16)).toEqual(['exact', 'stale_lease', 12, 7, null])+ })+})++describe('SQL contracts that a silent edit would otherwise break', () => {+ beforeEach(() => {+ vi.mocked(pool.query).mockReset()+ txClient.query.mockClear()+ vi.mocked(pool.query).mockResolvedValue({ rows: [], rowCount: 1 } as never)+ })++ it('refuses to promote a reserved attempt that already carries a credential jti (N-22)', async () => {+ // The JTI-free authorize-link path must never adopt an attempt that the+ // secret-ticket path already claimed. Dropping this predicate leaves the+ // whole suite green, so assert the clause itself.+ await promoteReservedOauthBrokerProviderAttempt(+ {+ id: 'attempt-1',+ invocationId: 'inv-1',+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'sdk-recipe',+ attemptGeneration: 1,+ attemptIndex: 1,+ model: 'gpt-5.1',+ targetRef: 'primary-codex',+ },+ pool+ )+ const [sql] = vi.mocked(pool.query).mock.calls[0] as unknown as [string, unknown[]]+ expect(sql).toContain('AND credential_jti IS NULL')+ expect(sql).toContain("AND status = 'reserved'")+ expect(sql).toContain("AND provider = 'codex-subscription'")+ })++ it('only revives an invocation that is persisted as failed', async () => {+ // The counterpart of finalize's double-charge guard: an oauth close whose+ // Codex row was already billed is persisted as provider_unavailable, and+ // this predicate is what makes that state non-revivable.+ await reviveFailedInvocation({+ id: 'inv-1',+ recipeNamespace: 'sandbox-recipes',+ recipeName: 'sdk-recipe',+ leaseSeconds: 180,+ })+ const [sql] = txClient.query.mock.calls[0] as unknown as [string, unknown[]]+ expect(sql).toContain("AND status = 'failed'")+ expect(sql).not.toMatch(/status\s+IN\s*\(/i)})})diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.tsindex 581875699..0cbc87744 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts@@ -19,8 +19,20 @@ function databaseUrl(baseUrl: string, database: string): string {return url.toString()}+/**+ * Every transaction in this suite runs as the PRODUCTION runtime role, not as+ * the superuser that creates the database. `plugin_workload_sdk_spend_outcomes`+ * is granted SELECT/INSERT only (access profile `append`), so a write outside+ * that envelope raises 42501 and fails the test. Running these paths as a+ * superuser is exactly how an UPDATE against the spend ledger shipped green.+ *+ * `SET LOCAL` is transaction-scoped, so a pooled connection can never leak the+ * role into a later checkout, and `CREATE DATABASE` / `initDb` / `DROP+ * DATABASE` stay superuser.+ */async function begin(client: PoolClient): Promise<void> {await client.query('BEGIN')+ await client.query('SET LOCAL ROLE control_api_runtime')}async function commit(client: PoolClient): Promise<void> {@@ -69,11 +81,25 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =}})+ const CODEX_TARGET = {+ targetRef: 'primary-codex',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+ }+ const OPENAI_TARGET = {+ targetRef: 'primary-openai',+ provider: 'openai',+ model: 'gpt-4o-mini',+ credentialSlot: 'openai-api-key',+ }+async function seedAttempt(client: PoolClient,options: {status?: PromptBridgeFinalizationInput['status']leaseExpired?: boolean+ codex?: boolean} = {}): Promise<PromptBridgeFinalizationInput> {const invocationId = randomUUID()@@ -84,12 +110,7 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =const leaseExpression = options.leaseExpired? "now() - interval '5 minutes'": "now() + interval '5 minutes'"- const target = {- targetRef: 'primary-openai',- provider: 'openai',- model: 'gpt-4o-mini',- credentialSlot: 'openai-api-key',- }+ const target = options.codex ? { ...CODEX_TARGET } : { ...OPENAI_TARGET }await client.query(`INSERT INTO plugin_workload_sdk_invocations(id, recipe_namespace, recipe_name, caller_ref, method, detail,@@ -151,6 +172,92 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =}}+ /** Link a Codex ledger row to an SDK attempt, optionally already usage-ready. */+ async function linkCodex(+ client: PoolClient,+ input: PromptBridgeFinalizationInput,+ options: { ready?: boolean; providerAttemptId?: string; attemptIndex?: number } = {}+ ): Promise<string> {+ const attempt = await insertLlmProviderAttempt(client, {+ callerKind: 'recipe',+ hostRef: input.hostRef,+ recipeNamespace: input.recipeNamespace,+ recipeName: input.recipeName,+ invocationId: input.invocationId,+ attemptGeneration: input.attemptGeneration,+ providerAttemptIndex: options.attemptIndex ?? input.providerAttemptIndex,+ model: CODEX_TARGET.model,+ requestHash: 'a'.repeat(64),+ policyRevision: 1,+ policyHash: 'b'.repeat(64),+ budgetReservationId: randomUUID(),+ connectionRevision: 1,+ pluginWorkloadSdkProviderAttemptId: options.providerAttemptId ?? input.providerAttemptId,+ })+ if (options.ready) await markCodexUsageReady(client, attempt.id)+ return attempt.id+ }++ /**+ * The state the Codex proxy finalize leaves behind on success. Written+ * directly because driving the whole ticket lifecycle is the dual-ledger+ * suite's job, not this one's.+ */+ async function markCodexUsageReady(+ client: PoolClient,+ codexAttemptId: string,+ tokens: [number, number] = [12, 7]+ ): Promise<void> {+ await client.query(+ `UPDATE llm_provider_attempts+ SET status = 'finalized', outcome = 'success',+ usage_input_tokens = $2, usage_output_tokens = $3, finalized_at = now()+ WHERE id = $1`,+ [codexAttemptId, tokens[0], tokens[1]]+ )+ }++ /** Insert an extra physical attempt, as a failover would have left behind. */+ async function seedPriorAttempt(+ client: PoolClient,+ input: PromptBridgeFinalizationInput,+ options: { attemptIndex: number; status: string; codex?: boolean }+ ): Promise<string> {+ const id = randomUUID()+ const target = options.codex ? CODEX_TARGET : OPENAI_TARGET+ await client.query(+ `INSERT INTO plugin_workload_sdk_provider_attempts+ (id, invocation_id, recipe_namespace, recipe_name, attempt_generation,+ attempt_index, target_ref, provider, model, credential_slot, status,+ completed_at)+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())`,+ [+ id,+ input.invocationId,+ input.recipeNamespace,+ input.recipeName,+ input.attemptGeneration,+ options.attemptIndex,+ target.targetRef,+ target.provider,+ target.model,+ target.credentialSlot,+ options.status,+ ]+ )+ return id+ }++ async function spendRows(providerAttemptId: string) {+ const result = await dbPool.query(+ `SELECT outcome, reason, input_tokens, output_tokens, usage_request_id, host_ref+ FROM plugin_workload_sdk_spend_outcomes+ WHERE provider_attempt_id = $1`,+ [providerAttemptId]+ )+ return result.rows as Array<Record<string, unknown>>+ }+it('writes one immutable unknown outcome and makes an identical replay idempotent', async () => {const client = await dbPool.connect()try {@@ -481,4 +588,330 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =client.release()}})++ it('never rewrites the sweeper floor; late Codex usage is derived at read time', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { codex: true, leaseExpired: true })+ // Out of the in-flight grace, so the sweeper is free to close it.+ const codexAttemptId = await linkCodex(client, input)+ await client.query(+ `UPDATE llm_provider_attempts SET created_at = now() - interval '30 minutes' WHERE id = $1`,+ [codexAttemptId]+ )+ expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)+ await commit(client)++ expect(await spendRows(input.providerAttemptId)).toEqual([+ {+ outcome: 'unknown',+ reason: 'stale_lease',+ input_tokens: null,+ output_tokens: null,+ usage_request_id: null,+ host_ref: null,+ },+ ])++ // Codex reports its usage after the sweep.+ const usageClient = await dbPool.connect()+ try {+ await begin(usageClient)+ await markCodexUsageReady(usageClient, codexAttemptId)+ await commit(usageClient)+ } finally {+ usageClient.release()+ }++ const replayClient = await dbPool.connect()+ try {+ await begin(replayClient)+ const replay = await finalizePromptBridgeInTransaction(+ {+ ...input,+ status: 'complete',+ usage: {+ llmSecretName: '',+ callerRef: 'integration-test',+ fallbackUsed: false,+ attemptCount: 1,+ inputTokens: 99,+ outputTokens: 99,+ },+ },+ replayClient+ )+ await commit(replayClient)+ // J3: the outcome reports the derived truth...+ expect(replay).toMatchObject({ outcome: 'exact', idempotent: true, usageAccepted: false })+ } finally {+ replayClient.release()+ }++ // ...while the floor itself is byte-for-byte what the sweeper wrote.+ // An UPDATE here would have been a 42501 under the runtime role.+ expect(await spendRows(input.providerAttemptId)).toEqual([+ {+ outcome: 'unknown',+ reason: 'stale_lease',+ input_tokens: null,+ output_tokens: null,+ usage_request_id: null,+ host_ref: null,+ },+ ])+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('lets the sweeper freeze exact when the linked Codex usage already landed', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { codex: true, leaseExpired: true })+ await linkCodex(client, input, { ready: true })+ expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)+ await commit(client)++ // Addendum A.4: a ready Codex row is the best floor provable without a+ // JWT, so the sweeper no longer under-reports it as unknown.+ expect(await spendRows(input.providerAttemptId)).toEqual([+ {+ outcome: 'exact',+ reason: 'stale_lease',+ input_tokens: 12,+ output_tokens: 7,+ usage_request_id: null,+ host_ref: null,+ },+ ])+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('closes a failed oauth attempt whose Codex row is ready as exact and non-revivable', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { codex: true, status: 'failed' })+ input.reason = 'provider_stream_failed'+ await linkCodex(client, input, { ready: true })+ // N-BLK-2(a): this used to persist `unknown` WITH tokens and die on+ // token_pair_check (23514 -> 500).+ const result = await finalizePromptBridgeInTransaction(input, client)+ await commit(client)++ expect(result).toMatchObject({ status: 'failed', outcome: 'exact', idempotent: false })+ expect(await spendRows(input.providerAttemptId)).toEqual([+ {+ outcome: 'exact',+ reason: 'provider_stream_failed',+ input_tokens: 12,+ output_tokens: 7,+ usage_request_id: null,+ host_ref: input.hostRef,+ },+ ])++ // Guard 1.a: reviveFailedInvocation only matches `status = 'failed'`, so+ // persisting provider_unavailable is what stops the same idempotency key+ // from launching a second billable Codex call.+ const state = await dbPool.query(+ `SELECT inv.status AS invocation_status,+ receipt.status AS receipt_status,+ attempt.status AS provider_status+ FROM plugin_workload_sdk_invocations inv+ JOIN plugin_workload_sdk_invocation_attempts receipt+ ON receipt.invocation_id = inv.id+ JOIN plugin_workload_sdk_provider_attempts attempt ON attempt.id = $1+ WHERE inv.id = $2`,+ [input.providerAttemptId, input.invocationId]+ )+ expect(state.rows[0]).toEqual({+ invocation_status: 'provider_unavailable',+ receipt_status: 'provider_unavailable',+ provider_status: 'provider_unavailable',+ })+ const revivable = await dbPool.query(+ `SELECT count(*)::int AS count+ FROM plugin_workload_sdk_invocations+ WHERE id = $1 AND status = 'failed'`,+ [input.invocationId]+ )+ expect(revivable.rows[0]?.count).toBe(0)+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('resolves three identical failed calls while Codex usage lands between them (N-05)', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { codex: true, status: 'failed' })+ const codexAttemptId = await linkCodex(client, input)+ await client.query(+ `UPDATE llm_provider_attempts SET status = 'finalized', outcome = 'unknown' WHERE id = $1`,+ [codexAttemptId]+ )+ const first = await finalizePromptBridgeInTransaction(input, client)+ await commit(client)+ expect(first).toMatchObject({ outcome: 'unknown', idempotent: false })++ const usageClient = await dbPool.connect()+ try {+ await begin(usageClient)+ await markCodexUsageReady(usageClient, codexAttemptId)+ await commit(usageClient)+ } finally {+ usageClient.release()+ }++ // Before the fix the second call rewrote the row to `exact` and the third+ // then 409'd against its own predecessor. The floor is immutable now, so+ // both replays agree.+ for (const _attempt of [2, 3]) {+ const replayClient = await dbPool.connect()+ try {+ await begin(replayClient)+ const replay = await finalizePromptBridgeInTransaction(input, replayClient)+ await commit(replayClient)+ expect(replay).toMatchObject({+ status: 'failed',+ outcome: 'exact',+ idempotent: true,+ usageAccepted: false,+ })+ } finally {+ replayClient.release()+ }+ }+ expect(await spendRows(input.providerAttemptId)).toEqual([+ {+ outcome: 'unknown',+ reason: 'integration_provider_unavailable',+ input_tokens: null,+ output_tokens: null,+ usage_request_id: null,+ host_ref: input.hostRef,+ },+ ])+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('settles the attempts a successful failover displaced (RP-539-003)', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { codex: true, status: 'complete' })+ // The winner is attempt 3; 1 and 2 are the terminal attempts a+ // successful failover leaves behind and never finalizes.+ await client.query(+ `UPDATE plugin_workload_sdk_provider_attempts SET attempt_index = 3 WHERE id = $1`,+ [input.providerAttemptId]+ )+ input.providerAttemptIndex = 3+ const failedId = await seedPriorAttempt(client, input, { attemptIndex: 1, status: 'failed' })+ const unavailableId = await seedPriorAttempt(client, input, {+ attemptIndex: 2,+ status: 'provider_unavailable',+ codex: true,+ })+ await linkCodex(client, input, {+ ready: true,+ providerAttemptId: unavailableId,+ attemptIndex: 2,+ })+ await linkCodex(client, input, { ready: true, attemptIndex: 3 })++ const result = await finalizePromptBridgeInTransaction(input, client)+ await commit(client)+ expect(result).toMatchObject({ outcome: 'exact', idempotent: false })++ expect(await spendRows(failedId)).toEqual([+ {+ outcome: 'not_executed',+ reason: 'prior_attempt_failed',+ input_tokens: null,+ output_tokens: null,+ usage_request_id: null,+ host_ref: input.hostRef,+ },+ ])+ // A displaced oauth attempt whose Codex row is ready still owes exact+ // spend, even though the host never reported it.+ expect(await spendRows(unavailableId)).toEqual([+ {+ outcome: 'exact',+ reason: 'prior_attempt_provider_unavailable',+ input_tokens: 12,+ output_tokens: 7,+ usage_request_id: null,+ host_ref: input.hostRef,+ },+ ])++ // A replay is pure reads: no second row, no rewrite.+ const replayClient = await dbPool.connect()+ try {+ await begin(replayClient)+ await expect(finalizePromptBridgeInTransaction(input, replayClient)).resolves.toMatchObject(+ { idempotent: true, outcome: 'exact' }+ )+ await commit(replayClient)+ } finally {+ replayClient.release()+ }+ const total = await dbPool.query(+ `SELECT count(*)::int AS count+ FROM plugin_workload_sdk_spend_outcomes+ WHERE invocation_id = $1`,+ [input.invocationId]+ )+ expect(total.rows[0]?.count).toBe(3)+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })++ it('fails loudly when a displaced attempt is not terminal', async () => {+ const client = await dbPool.connect()+ try {+ await begin(client)+ const input = await seedAttempt(client, { status: 'complete' })+ await client.query(+ `UPDATE plugin_workload_sdk_provider_attempts SET attempt_index = 2 WHERE id = $1`,+ [input.providerAttemptId]+ )+ input.providerAttemptIndex = 2+ await seedPriorAttempt(client, input, { attemptIndex: 1, status: 'in_progress' })++ await expect(finalizePromptBridgeInTransaction(input, client)).rejects.toThrow(+ /is not terminal \(status=in_progress\); the reservation fence is broken/+ )+ await client.query('ROLLBACK')++ // The whole transaction rolled back, so no half-settled ledger survives.+ const total = await dbPool.query(+ `SELECT count(*)::int AS count+ FROM plugin_workload_sdk_spend_outcomes+ WHERE invocation_id = $1`,+ [input.invocationId]+ )+ expect(total.rows[0]?.count).toBe(0)+ } finally {+ await client.query('ROLLBACK').catch(() => undefined)+ client.release()+ }+ })})diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.test.tsindex 13129c6e7..bec9015d9 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts@@ -1,6 +1,7 @@import { describe, expect, it, vi } from 'vitest'import {PromptBridgeFinalizationError,+ deriveEffectiveSpend,finalizePromptBridgeInTransaction,} from '../src/services/pluginWorkloadSdkFinalization.js'@@ -43,6 +44,134 @@ function dbWithRows(...rows: unknown[]) {return { query }}+const CODEX_TARGET = {+ targetRef: 'codex-primary',+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ credentialSlot: '',+} as const++/**+ * The row PostgreSQL echoes from the spend-ledger INSERT's RETURNING clause.+ * The finalizer derives its reported outcome from this row, so a mock that+ * returns a placeholder would hide the derivation instead of exercising it.+ */+function spendRow(overrides: Record<string, unknown> = {}) {+ return {+ provider_attempt_id: IDS.providerAttempt,+ invocation_id: IDS.invocation,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ attempt_generation: 1,+ attempt_index: 1,+ target_ref: inputBase.target.targetRef,+ host_ref: inputBase.hostRef,+ provider: inputBase.target.provider,+ model: inputBase.target.model,+ credential_slot: inputBase.target.credentialSlot,+ outcome: 'unknown',+ input_tokens: null,+ output_tokens: null,+ ...overrides,+ }+}++function codexSpendRow(overrides: Record<string, unknown> = {}) {+ return spendRow({+ target_ref: CODEX_TARGET.targetRef,+ provider: CODEX_TARGET.provider,+ model: CODEX_TARGET.model,+ credential_slot: CODEX_TARGET.credentialSlot,+ ...overrides,+ })+}++/** A linked `llm_provider_attempts` row as the store maps it back. */+function linkedCodexRow(overrides: Record<string, unknown> = {}) {+ return {+ id: '33333333-3333-4333-8333-333333333333',+ caller_kind: 'recipe',+ host_ref: inputBase.hostRef,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ invocation_id: IDS.invocation,+ attempt_generation: 1,+ provider_attempt_index: 1,+ provider: 'codex-subscription',+ model: 'gpt-5.1',+ request_hash: 'hash',+ policy_revision: 1,+ policy_hash: 'a'.repeat(64),+ budget_reservation_id: 'budget-1',+ connection_revision: 1,+ plugin_workload_sdk_provider_attempt_id: IDS.providerAttempt,+ status: 'finalized',+ outcome: 'success',+ usage_input_tokens: 12,+ usage_output_tokens: 7,+ created_at: new Date(),+ ...overrides,+ }+}++/** invocations / invocation_attempts / provider_attempts rows for the fence. */+function fenceRows(providerAttempt: Record<string, unknown> = {}) {+ return [+ {+ id: IDS.invocation,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ method: 'promptBridge',+ status: 'in_progress',+ attempt_generation: 1,+ },+ {+ invocation_id: IDS.invocation,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ attempt_generation: 1,+ method: 'promptBridge',+ status: 'in_progress',+ },+ {+ id: IDS.providerAttempt,+ invocation_id: IDS.invocation,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ attempt_generation: 1,+ attempt_index: 1,+ target_ref: inputBase.target.targetRef,+ host_ref: inputBase.hostRef,+ provider: inputBase.target.provider,+ model: inputBase.target.model,+ credential_slot: inputBase.target.credentialSlot,+ status: 'in_progress',+ ...providerAttempt,+ },+ ]+}++function statementsOf(db: { query: { mock: { calls: unknown[][] } } }): string[] {+ return db.query.mock.calls.map(call => String(call[0]))+}++function insertParams(db: { query: { mock: { calls: unknown[][] } } }): unknown[] | undefined {+ const call = db.query.mock.calls.find(([statement]) =>+ String(statement).includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ )+ return call?.[1] as unknown[] | undefined+}++function updateParams(+ db: { query: { mock: { calls: unknown[][] } } },+ table: string+): unknown[] | undefined {+ const call = db.query.mock.calls.find(([statement]) =>+ String(statement).includes(`UPDATE ${table}`)+ )+ return call?.[1] as unknown[] | undefined+}+describe('Plugin Workload SDK promptBridge finalization', () => {it('atomically records exact usage and terminalizes the physical/logical receipts', async () => {const usageEvent = { request_id: IDS.providerAttempt }@@ -87,7 +216,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ spendRow({ outcome: 'exact', input_tokens: 12, output_tokens: 7 }))const result = await finalizePromptBridgeInTransaction(@@ -169,7 +298,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ spendRow())const result = await finalizePromptBridgeInTransaction(@@ -245,7 +374,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ spendRow({ outcome: 'not_executed' }))const result = await finalizePromptBridgeInTransaction(@@ -320,7 +449,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ spendRow())await expect(@@ -450,17 +579,11 @@ describe('Plugin Workload SDK promptBridge finalization', () => {credential_slot: '',status: 'in_progress',},- {- id: '33333333-3333-4333-8333-333333333333',- status: 'finalized',- outcome: 'success',- usage_input_tokens: 12,- usage_output_tokens: 7,- },+ linkedCodexRow(),{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ codexSpendRow({ outcome: 'exact', input_tokens: 12, output_tokens: 7 }))const result = await finalizePromptBridgeInTransaction(@@ -633,7 +756,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ codexSpendRow())const result = await finalizePromptBridgeInTransaction(@@ -682,8 +805,12 @@ describe('Plugin Workload SDK promptBridge finalization', () => {'provider_completed',null,null,- IDS.providerAttempt,+ // N-17: Codex spend is ingested by the proxy finalize, not here, so no+ // usage_events row exists for this attempt and the FK-shaped column must+ // stay NULL rather than point at a row that was never written.+ null,])+ expect(updateParams(db, 'plugin_workload_sdk_provider_attempts')?.[2]).toBe(false)})it('records Codex complete as unknown when linked success finalized without usage', async () => {@@ -747,7 +874,7 @@ describe('Plugin Workload SDK promptBridge finalization', () => {{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },- { rows: [], rowCount: 1 }+ codexSpendRow())const result = await finalizePromptBridgeInTransaction(@@ -830,60 +957,13 @@ describe('Plugin Workload SDK promptBridge finalization', () => {})})- it('upgrades a Codex unknown spend row to exact when linked usage becomes ready', async () => {- const row = {- provider_attempt_id: IDS.providerAttempt,- invocation_id: IDS.invocation,- recipe_namespace: inputBase.recipeNamespace,- recipe_name: inputBase.recipeName,- attempt_generation: 1,- attempt_index: 1,- target_ref: 'codex-primary',- host_ref: inputBase.hostRef,- provider: 'codex-subscription',- model: 'gpt-5.1',- credential_slot: '',- outcome: 'unknown',- input_tokens: null,- output_tokens: null,- }- const db = dbWithRows(- [],- row,- {- id: '33333333-3333-4333-8333-333333333333',- caller_kind: 'recipe',- host_ref: inputBase.hostRef,- recipe_namespace: inputBase.recipeNamespace,- recipe_name: inputBase.recipeName,- invocation_id: IDS.invocation,- attempt_generation: 1,- provider_attempt_index: 1,- provider: 'codex-subscription',- model: 'gpt-5.1',- request_hash: 'hash',- policy_revision: 1,- policy_hash: 'a'.repeat(64),- budget_reservation_id: 'budget-1',- connection_revision: 1,- status: 'finalized',- outcome: 'success',- usage_input_tokens: 12,- usage_output_tokens: 7,- created_at: new Date(),- },- { rows: [{ provider_attempt_id: IDS.providerAttempt }], rowCount: 1 }- )+ it('derives exact from linked Codex usage on replay without writing', async () => {+ const db = dbWithRows([], codexSpendRow(), linkedCodexRow())await expect(finalizePromptBridgeInTransaction({...inputBase,- target: {- targetRef: 'codex-primary',- provider: 'codex-subscription',- model: 'gpt-5.1',- credentialSlot: '',- },+ target: { ...CODEX_TARGET },status: 'complete',usage: {llmSecretName: '',@@ -897,14 +977,21 @@ describe('Plugin Workload SDK promptBridge finalization', () => {db as never)).resolves.toMatchObject({- idempotent: false,+ idempotent: true,outcome: 'exact',usageAccepted: false,})- const upgrade = db.query.mock.calls.find(([statement]: [string]) =>- statement.includes("SET outcome = 'exact'")+ // N-BLK-1: the floor is immutable. The runtime role holds SELECT/INSERT+ // only, so any write here would be a 42501 in production.+ const statements = statementsOf(db)+ expect(statements.some(sql => /UPDATE plugin_workload_sdk_spend_outcomes/i.test(sql))).toBe(+ false)- expect(upgrade?.[1]).toEqual([IDS.providerAttempt, 12, 7])+ expect(+ statements.some(sql => /INSERT INTO plugin_workload_sdk_spend_outcomes/i.test(sql))+ ).toBe(false)+ // advisory lock → floor SELECT → linked Codex row. Nothing else.+ expect(statements).toHaveLength(3)})it('refuses to replay a sweeper unknown while the linked Codex attempt is still in flight', async () => {@@ -1151,3 +1238,487 @@ describe('Plugin Workload SDK promptBridge finalization', () => {})})})++// ─── Derive-on-read: the floor is immutable, the truth is derived ─────────++const codexInput = {+ ...inputBase,+ target: { ...CODEX_TARGET },+} as const++function codexFence(providerAttemptStatus: string) {+ return fenceRows({+ target_ref: CODEX_TARGET.targetRef,+ provider: CODEX_TARGET.provider,+ model: CODEX_TARGET.model,+ credential_slot: CODEX_TARGET.credentialSlot,+ status: providerAttemptStatus,+ })+}++describe('spend floor writers (Addendum A.4 "best-floor")', () => {+ it.each([+ ['failed', 'provider_unavailable'],+ ['provider_unavailable', 'provider_unavailable'],+ ] as const)(+ 'freezes an exact floor for a %s oauth close whose linked Codex usage is ready',+ async (status, persistedStatus) => {+ ingest.mockReset()+ project.mockReset()+ const db = dbWithRows(+ [],+ [],+ ...codexFence('in_progress'),+ linkedCodexRow(),+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow({ outcome: 'exact', input_tokens: 12, output_tokens: 7 })+ )++ const result = await finalizePromptBridgeInTransaction(+ { ...codexInput, status, reason: 'provider_stream_failed' },+ db as never+ )++ // N-BLK-2(a): before the fix this persisted `unknown` WITH tokens and+ // died on token_pair_check (23514 -> 500).+ expect(insertParams(db)?.slice(11, 16)).toEqual([+ 'exact',+ 'provider_stream_failed',+ 12,+ 7,+ null,+ ])+ // The wire status still echoes what the host declared; controlApiClient+ // rejects the response otherwise.+ expect(result).toMatchObject({ status, outcome: 'exact', idempotent: false })+ // Guard 1.a: the persisted terminal state is provider_unavailable, which+ // reviveFailedInvocation (WHERE status = 'failed') cannot revive, so the+ // same idempotency key cannot launch a second billable Codex call.+ expect(updateParams(db, 'plugin_workload_sdk_provider_attempts')?.[1]).toBe(persistedStatus)+ expect(updateParams(db, 'plugin_workload_sdk_invocation_attempts')?.[1]).toBe(persistedStatus)+ expect(updateParams(db, 'plugin_workload_sdk_invocations')?.[1]).toBe(persistedStatus)+ expect(ingest).not.toHaveBeenCalled()+ }+ )++ it('keeps a provider_unavailable close revivable-free without renaming a failed close that Codex never billed', async () => {+ ingest.mockReset()+ project.mockReset()+ const db = dbWithRows(+ [],+ [],+ ...codexFence('in_progress'),+ [],+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow({ outcome: 'not_executed' })+ )++ const result = await finalizePromptBridgeInTransaction(+ { ...codexInput, status: 'failed', reason: 'no_grant' },+ db as never+ )++ // No linked Codex row at all: nothing was billed, so the close stays+ // `failed` and the invocation stays revivable on a fresh attempt.+ expect(result).toMatchObject({ status: 'failed', outcome: 'not_executed' })+ expect(updateParams(db, 'plugin_workload_sdk_invocations')?.[1]).toBe('failed')+ })++ it('refuses to promote a Codex row that errored with usage or reported a partial token pair', async () => {+ for (const linked of [+ linkedCodexRow({ outcome: 'error' }),+ linkedCodexRow({ usage_output_tokens: null }),+ ]) {+ ingest.mockReset()+ project.mockReset()+ const db = dbWithRows(+ [],+ [],+ ...codexFence('in_progress'),+ linked,+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow()+ )++ const result = await finalizePromptBridgeInTransaction(+ {+ ...codexInput,+ status: 'complete',+ usage: {+ llmSecretName: '',+ callerRef: 'scanner',+ fallbackUsed: false,+ attemptCount: 1,+ inputTokens: 4,+ outputTokens: 2,+ },+ },+ db as never+ )++ // N-BLK-2(b)/(c): `unknown` never carries tokens, and the host's own+ // usage claim is discarded for an oauth-broker attempt.+ expect(insertParams(db)?.slice(11, 16)).toEqual([+ 'unknown',+ 'provider_completed',+ null,+ null,+ null,+ ])+ expect(result).toMatchObject({ outcome: 'unknown', usageAccepted: false })+ }+ })+})++describe('spend replay is read-only and judged against the floor', () => {+ it('reports exact for every replay status once the linked Codex usage landed', async () => {+ for (const status of ['complete', 'failed', 'provider_unavailable'] as const) {+ const db = dbWithRows([], codexSpendRow(), linkedCodexRow())+ const usage =+ status === 'complete'+ ? {+ usage: {+ llmSecretName: '',+ callerRef: 'scanner',+ fallbackUsed: false,+ attemptCount: 1,+ inputTokens: 4,+ outputTokens: 2,+ },+ }+ : {}+ // N-05: the third identical `failed` call used to 409 because the second+ // had rewritten the row to `exact`. The floor never moves, so the verdict+ // is stable and the reported outcome still catches up.+ await expect(+ finalizePromptBridgeInTransaction({ ...codexInput, status, ...usage }, db as never)+ ).resolves.toMatchObject({ status, outcome: 'exact', idempotent: true, usageAccepted: false })+ expect(statementsOf(db).some(sql => /plugin_workload_sdk_spend_outcomes/i.test(sql))).toBe(+ true+ )+ expect(+ statementsOf(db).some(sql =>+ /(UPDATE|INSERT INTO) plugin_workload_sdk_spend_outcomes/i.test(sql)+ )+ ).toBe(false)+ }+ })++ it.each(['failed', 'provider_unavailable'] as const)(+ 'still answers ledger_pending for a %s replay while the linked Codex attempt is in flight',+ async status => {+ // Regression cover for e13b4de8: ledger_pending is decided by the floor+ // plus the Codex row, never by the caller-chosen status. Reverting that+ // fix must turn this red.+ await expect(+ finalizePromptBridgeInTransaction(+ { ...codexInput, status },+ dbWithRows(+ [],+ codexSpendRow(),+ linkedCodexRow({+ status: 'authorized',+ outcome: null,+ usage_input_tokens: null,+ usage_output_tokens: null,+ })+ ) as never+ )+ ).rejects.toMatchObject({ code: 'ledger_pending', httpStatus: 409, retryable: true })+ }+ )++ it.each([+ [+ 'in flight',+ { status: 'authorized', outcome: null, usage_input_tokens: null, usage_output_tokens: null },+ ],+ ['usage-ready', {}],+ ] as const)(+ 'answers conflict, not the %s ledger state, when the JWT binding does not match the floor',+ async (_label, linkedOverrides) => {+ // N-07: the binding check runs before any state oracle, so a foreign+ // recipe cannot distinguish ledger_pending from exact from unknown.+ await expect(+ finalizePromptBridgeInTransaction(+ { ...codexInput, status: 'provider_unavailable' },+ dbWithRows(+ [],+ codexSpendRow({ recipe_namespace: 'someone-elses-namespace' }),+ linkedCodexRow(linkedOverrides)+ ) as never+ )+ ).rejects.toMatchObject({ code: 'conflict', httpStatus: 409 })+ }+ )++ it('never derives a not_executed floor, whatever the linked Codex row says', async () => {+ await expect(+ finalizePromptBridgeInTransaction(+ { ...codexInput, status: 'failed' },+ dbWithRows([], codexSpendRow({ outcome: 'not_executed' }), linkedCodexRow()) as never+ )+ ).resolves.toMatchObject({ outcome: 'not_executed', idempotent: true })+ })++ it.each(['complete', 'provider_unavailable'] as const)(+ 'conflicts when a %s replay contradicts a not_executed floor',+ async status => {+ const usage =+ status === 'complete'+ ? {+ usage: {+ llmSecretName: '',+ callerRef: 'scanner',+ fallbackUsed: false,+ attemptCount: 1,+ inputTokens: 4,+ outputTokens: 2,+ },+ }+ : {}+ await expect(+ finalizePromptBridgeInTransaction(+ { ...codexInput, status, ...usage },+ dbWithRows([], codexSpendRow({ outcome: 'not_executed' }), linkedCodexRow()) as never+ )+ ).rejects.toMatchObject({ code: 'conflict', httpStatus: 409 })+ }+ )+})++describe('deriveEffectiveSpend', () => {+ const codexFloor = (outcome: string, tokens: [number, number] | null = null) => ({+ provider: 'codex-subscription',+ outcome: outcome as 'exact' | 'unknown' | 'not_executed',+ input_tokens: tokens?.[0] ?? null,+ output_tokens: tokens?.[1] ?? null,+ })+ const ready = { outcome: 'success' as const, usageInputTokens: 12, usageOutputTokens: 7 }++ it.each([+ [+ 'unknown floor + ready Codex derives exact',+ codexFloor('unknown'),+ ready,+ { outcome: 'exact', inputTokens: 12, outputTokens: 7 },+ ],+ [+ 'unknown floor + no link stays unknown',+ codexFloor('unknown'),+ null,+ { outcome: 'unknown', inputTokens: null, outputTokens: null },+ ],+ [+ 'unknown floor + errored Codex stays unknown',+ codexFloor('unknown'),+ { outcome: 'error' as const, usageInputTokens: 12, usageOutputTokens: 7 },+ { outcome: 'unknown', inputTokens: null, outputTokens: null },+ ],+ [+ 'unknown floor + partial usage stays unknown',+ codexFloor('unknown'),+ { outcome: 'success' as const, usageInputTokens: 12, usageOutputTokens: null },+ { outcome: 'unknown', inputTokens: null, outputTokens: null },+ ],+ [+ 'not_executed is terminal',+ codexFloor('not_executed'),+ ready,+ { outcome: 'not_executed', inputTokens: null, outputTokens: null },+ ],+ [+ 'exact keeps its own tokens',+ codexFloor('exact', [3, 1]),+ ready,+ { outcome: 'exact', inputTokens: 3, outputTokens: 1 },+ ],+ [+ 'a non-oauth unknown floor is never derived',+ { provider: 'openai', outcome: 'unknown' as const, input_tokens: null, output_tokens: null },+ ready,+ { outcome: 'unknown', inputTokens: null, outputTokens: null },+ ],+ ])('%s', (_label, persisted, linked, expected) => {+ expect(deriveEffectiveSpend(persisted, linked)).toEqual(expected)+ })++ it('fails loudly on an exact floor without a token pair', () => {+ expect(() => deriveEffectiveSpend(codexFloor('exact'), null)).toThrow(+ /exact spend outcome persisted without a token pair/+ )+ })+})++describe('prior provider attempt settlement (RP-539-003)', () => {+ const PRIOR_ATTEMPT_ID = '44444444-4444-4444-8444-444444444444'++ function priorAttemptRow(status: string) {+ return {+ id: PRIOR_ATTEMPT_ID,+ recipe_namespace: inputBase.recipeNamespace,+ recipe_name: inputBase.recipeName,+ attempt_index: 1,+ target_ref: 'primary-openai',+ provider: 'openai',+ model: 'gpt-5.4-mini',+ credential_slot: 'openai-api-key',+ status,+ }+ }++ function failoverDb(priorStatus: string) {+ return dbWithRows(+ [],+ [],+ ...fenceRows({+ attempt_index: 2,+ target_ref: CODEX_TARGET.targetRef,+ provider: CODEX_TARGET.provider,+ model: CODEX_TARGET.model,+ credential_slot: CODEX_TARGET.credentialSlot,+ }),+ linkedCodexRow(),+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow({ attempt_index: 2, outcome: 'exact', input_tokens: 12, output_tokens: 7 }),+ [priorAttemptRow(priorStatus)],+ [],+ spendRow({ provider_attempt_id: PRIOR_ATTEMPT_ID, outcome: 'not_executed' })+ )+ }++ const winner = {+ ...codexInput,+ providerAttemptIndex: 2,+ status: 'complete' as const,+ usage: {+ llmSecretName: '',+ callerRef: 'scanner',+ fallbackUsed: false,+ attemptCount: 2,+ inputTokens: 4,+ outputTokens: 2,+ },+ }++ it.each([+ ['failed', 'not_executed'],+ ['skipped', 'not_executed'],+ ['provider_unavailable', 'unknown'],+ ] as const)(+ 'freezes a %s floor for the displaced attempt a successful failover never finalizes',+ async (priorStatus, expectedOutcome) => {+ const db = failoverDb(priorStatus)+ await expect(finalizePromptBridgeInTransaction(winner, db as never)).resolves.toMatchObject({+ outcome: 'exact',+ idempotent: false,+ })++ const inserts = db.query.mock.calls.filter(([statement]: [string]) =>+ statement.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ )+ expect(inserts).toHaveLength(2)+ const prior = inserts[1]?.[1] as unknown[]+ expect(prior?.[0]).toBe(PRIOR_ATTEMPT_ID)+ expect(prior?.[5]).toBe(1)+ expect(prior?.[7]).toBe(inputBase.hostRef)+ expect(prior?.slice(11, 16)).toEqual([+ expectedOutcome,+ `prior_attempt_${priorStatus}`,+ null,+ null,+ null,+ ])+ // The displaced attempt gets a ledger row only. Its invocation and+ // receipt belong to the winner's transition.+ const discovery = db.query.mock.calls.find(([statement]: [string]) =>+ statement.includes('attempt_index < $3')+ )+ expect(discovery?.[0]).toMatch(/FOR UPDATE/i)+ expect(discovery?.[1]).toEqual([IDS.invocation, 1, 2])+ }+ )++ it('freezes an exact floor for a displaced oauth attempt whose Codex row is ready', async () => {+ const db = dbWithRows(+ [],+ [],+ ...fenceRows({+ attempt_index: 2,+ target_ref: CODEX_TARGET.targetRef,+ provider: CODEX_TARGET.provider,+ model: CODEX_TARGET.model,+ credential_slot: CODEX_TARGET.credentialSlot,+ }),+ linkedCodexRow(),+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow({ attempt_index: 2, outcome: 'exact', input_tokens: 12, output_tokens: 7 }),+ [+ {+ ...priorAttemptRow('provider_unavailable'),+ target_ref: CODEX_TARGET.targetRef,+ provider: CODEX_TARGET.provider,+ model: CODEX_TARGET.model,+ credential_slot: CODEX_TARGET.credentialSlot,+ },+ ],+ linkedCodexRow({+ id: '55555555-5555-4555-8555-555555555555',+ plugin_workload_sdk_provider_attempt_id: PRIOR_ATTEMPT_ID,+ usage_input_tokens: 5,+ usage_output_tokens: 3,+ }),+ spendRow({ provider_attempt_id: PRIOR_ATTEMPT_ID, outcome: 'exact' })+ )++ await finalizePromptBridgeInTransaction(winner, db as never)++ const inserts = db.query.mock.calls.filter(([statement]: [string]) =>+ statement.includes('INSERT INTO plugin_workload_sdk_spend_outcomes')+ )+ expect((inserts[1]?.[1] as unknown[])?.slice(11, 16)).toEqual([+ 'exact',+ 'prior_attempt_provider_unavailable',+ 5,+ 3,+ null,+ ])+ })++ it('fails loudly when a displaced attempt is not terminal', async () => {+ // reservePluginWorkloadSdkProviderAttempt cannot hand out index 2 over a+ // non-terminal index 1, so this is a broken fence, not a tolerable case.+ const db = failoverDb('in_progress')+ await expect(finalizePromptBridgeInTransaction(winner, db as never)).rejects.toThrow(+ /is not terminal \(status=in_progress\); the reservation fence is broken/+ )+ })++ it('does not query for predecessors of the first attempt', async () => {+ const db = dbWithRows(+ [],+ [],+ ...codexFence('in_progress'),+ linkedCodexRow(),+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ { rows: [], rowCount: 1 },+ codexSpendRow({ outcome: 'exact', input_tokens: 12, output_tokens: 7 })+ )+ await finalizePromptBridgeInTransaction(+ { ...codexInput, status: 'provider_unavailable' },+ db as never+ )+ expect(statementsOf(db).some(sql => sql.includes('attempt_index < $3'))).toBe(false)+ })+})diff --git a/control-api/test/services.pluginWorkloadSdkSchema.test.ts b/control-api/test/services.pluginWorkloadSdkSchema.test.tsindex e30bb0cea..415546d34 100644--- a/control-api/test/services.pluginWorkloadSdkSchema.test.ts+++ b/control-api/test/services.pluginWorkloadSdkSchema.test.ts@@ -1,8 +1,16 @@import { describe, expect, it } from 'vitest'-import { readFileSync } from 'node:fs'+import { readFileSync, readdirSync } from 'node:fs'import { dirname, join } from 'node:path'import { fileURLToPath } from 'node:url'+function sourceFiles(root: string): string[] {+ return readdirSync(root, { withFileTypes: true }).flatMap(entry => {+ const full = join(root, entry.name)+ if (entry.isDirectory()) return sourceFiles(full)+ return entry.isFile() && full.endsWith('.ts') ? [full] : []+ })+}+describe('pluginWorkloadSdkSchema', () => {it('scopes quota counters by recipe namespace in the primary key', () => {const schemaPath = join(@@ -75,4 +83,31 @@ describe('pluginWorkloadSdkSchema', () => {expect(source).toMatch(/plugin_workload_sdk_spend_outcomes_token_pair_check/)expect(source).toMatch(/addPluginWorkloadSdkSpendOutcomeLedger/)})++ it('writes the spend ledger from exactly one INSERT and never updates or deletes it', () => {+ // The runtime role holds SELECT/INSERT only (access profile `append`), so+ // any UPDATE/DELETE here is a 42501 in production that no superuser test+ // can see. Pinning a single INSERT site also keeps token_pair_check+ // guaranteed by the PersistableSpend type in every writer.+ const srcRoot = join(dirname(fileURLToPath(import.meta.url)), '../src')+ const files = sourceFiles(srcRoot)+ expect(files.length).toBeGreaterThan(0)++ const inserts: string[] = []+ const mutations: string[] = []+ for (const file of files) {+ const source = readFileSync(file, 'utf8')+ for (const _ of source.matchAll(/INSERT INTO plugin_workload_sdk_spend_outcomes/g)) {+ inserts.push(file)+ }+ if (+ /UPDATE plugin_workload_sdk_spend_outcomes/.test(source) ||+ /DELETE FROM plugin_workload_sdk_spend_outcomes/.test(source)+ ) {+ mutations.push(file)+ }+ }+ expect(mutations).toEqual([])+ expect(inserts).toEqual([join(srcRoot, 'services/pluginWorkloadSdkSpendLedger.ts')])+ })})