Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): hold budget reservations and usage grace for the whole attempt
Alfredo Lopez8ed35d13983b
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex f64de5231..716800e60 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 20033d94b..a4c6f4544 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.600",+ "version": "0.1.601","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex 41ddda9ca..fa82b204e 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -909,6 +909,9 @@ export const config: Config = {// Danger-zone reservation TTL (§9.8a: ~2-3× the rollup lag, ~5 min). Short// enough that a hung reservation auto-frees; long enough that real spend has// reached the rollups before it expires (no double-count on the next check).+ // This is the task-level TTL. A Codex or Grok provider attempt reserves for+ // its whole lifetime plus this value, and the in-flight usage grace adds it+ // to the longest attempt (services/llmProviderAttemptEnvelope.ts).budgetReservationTtlSeconds: positiveIntegerFromEnv('BUDGET_RESERVATION_TTL_SECONDS', 300),// Default 180 days. Archival runs daily at 02:00 UTC; older terminal// approvals move to the archive table.diff --git a/control-api/src/services/budgets/check.ts b/control-api/src/services/budgets/check.tsindex cd15359e1..63313e341 100644--- a/control-api/src/services/budgets/check.ts+++ b/control-api/src/services/budgets/check.ts@@ -100,6 +100,8 @@ export type BudgetCheckOptions = {requiredUnit?: 'tokens'/** When set, danger-zone reservations join this already-open transaction. */transactionClient?: ReservationTxClient+ /** Danger-zone reservation TTL in seconds; absent keeps the config TTL. */+ reservationTtlSeconds?: number}export type BudgetCheckResult = {@@ -413,6 +415,7 @@ export async function evaluateBudgetCheck(// free this host's own reservations (§5.4). Already normalized// above and bound to claims.hostRefs[0] by the route's claim-binding.hostRef: dimensions.host_ref ?? null,+ ttlSeconds: options.reservationTtlSeconds,},connector,options.transactionClientdiff --git a/control-api/src/services/budgets/reservations.ts b/control-api/src/services/budgets/reservations.tsindex 3155a879c..2501a2ab1 100644--- a/control-api/src/services/budgets/reservations.ts+++ b/control-api/src/services/budgets/reservations.ts@@ -66,6 +66,12 @@ export type DangerZoneReserveInput = {* by the TTL sweep, never by an early release.*/hostRef: string | null+ /**+ * Reservation lifetime in seconds. Absent for a task-level check, which+ * keeps BUDGET_RESERVATION_TTL_SECONDS; an LLM provider attempt passes its+ * own, because its spend arrives only when a stream of up to 30 min ends.+ */+ ttlSeconds?: number}export type DangerZoneReserveResult =@@ -145,7 +151,13 @@ async function decideDangerZoneReservation(`INSERT INTO budget_pending_reservations (budget_id, est_amount, task_ref, host_ref, expires_at)VALUES ($1, $2, $3, $4, NOW() + ($5::int * INTERVAL '1 second'))RETURNING id`,- [input.budgetId, input.estAmount, input.taskRef, input.hostRef, BUDGET_RESERVATION_TTL_SECONDS]+ [+ input.budgetId,+ input.estAmount,+ input.taskRef,+ input.hostRef,+ input.ttlSeconds ?? BUDGET_RESERVATION_TTL_SECONDS,+ ])const reservationId = String((insertRes.rows[0] as { id: unknown }).id)return { decision: 'allow', reservationId }diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.tsindex 8c759a278..364dfe432 100644--- a/control-api/src/services/grokProviderAttemptRedemption.ts+++ b/control-api/src/services/grokProviderAttemptRedemption.ts@@ -13,6 +13,7 @@ import {loadGrokSubscriptionSecrets,} from './grokSubscriptionConnection.js'import { GrokSubscriptionOAuthError, ensureFreshGrokAccessToken } from './grokSubscriptionOAuth.js'+import { GROK_MAX_STREAM_DURATION_MS } from './llmProviderAttemptEnvelope.js'import { opaqueAttemptReceipt } from './llmProviderAttemptRedemption.js'import {loadLlmProviderAttempt,@@ -26,7 +27,7 @@ const log = rootLogger.child({ module: 'grok-provider-attempt-redemption' })export const GROK_COMPLETIONS_ORIGIN = COMPLETIONS_ORIGINexport const GROK_CATALOG_ORIGIN = CATALOG_ORIGINexport const GROK_TRANSPORT_PROTOCOL = TRANSPORT_PROTOCOL_VERSION-export const GROK_MAX_STREAM_DURATION_MS = 1_800_000+export { GROK_MAX_STREAM_DURATION_MS }export type GrokProviderAttemptRedeemErrorCode =| 'disabled'diff --git a/control-api/src/services/grokProviderAttemptTicket.ts b/control-api/src/services/grokProviderAttemptTicket.tsindex 9bcc1ac0c..ae36a4606 100644--- a/control-api/src/services/grokProviderAttemptTicket.ts+++ b/control-api/src/services/grokProviderAttemptTicket.ts@@ -2,11 +2,12 @@ import jwt from 'jsonwebtoken'import { createPublicKey, randomUUID } from 'node:crypto'import { config } from '../config.js'import type { DbClient } from '../db.js'+import { GROK_EXECUTION_TICKET_TTL_SECONDS } from './llmProviderAttemptEnvelope.js'import { registerLlmProviderAttemptTicket } from './llmProviderAttemptStore.js'export const GROK_EXECUTION_TICKET_TYP = 'grok-execution-ticket' as constexport const GROK_EXECUTION_TICKET_AUDIENCE = 'grok-llm-proxy'-export const GROK_EXECUTION_TICKET_TTL_SECONDS = 60+export { GROK_EXECUTION_TICKET_TTL_SECONDS }export type GrokExecutionTicketClaims = {jti: stringdiff --git a/control-api/src/services/llmProviderAttemptAuthorizer.ts b/control-api/src/services/llmProviderAttemptAuthorizer.tsindex e428f1a88..051b0fcc6 100644--- a/control-api/src/services/llmProviderAttemptAuthorizer.ts+++ b/control-api/src/services/llmProviderAttemptAuthorizer.ts@@ -31,6 +31,10 @@ import {getSafeGrokSubscriptionConnection,isGrokUnassignedConnectionKey,} from './grokSubscriptionConnection.js'+import {+ CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ GROK_ATTEMPT_RESERVATION_TTL_SECONDS,+} from './llmProviderAttemptEnvelope.js'import {getMaxLlmProviderAttemptGeneration,insertLlmProviderAttempt,@@ -452,7 +456,11 @@ async function authorizeGrokProviderAttempt(},db,{ connect: async () => tx as never },- { requiredUnit: 'tokens', transactionClient: tx }+ {+ requiredUnit: 'tokens',+ transactionClient: tx,+ reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,+ })if (!budget.allowed) {throw new LlmProviderAttemptAuthorizeError(@@ -835,7 +843,11 @@ export async function authorizeLlmProviderAttempt(},db,{ connect: async () => tx as never },- { requiredUnit: 'tokens', transactionClient: tx }+ {+ requiredUnit: 'tokens',+ transactionClient: tx,+ reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ })if (!budget.allowed) {throw new LlmProviderAttemptAuthorizeError(diff --git a/control-api/src/services/llmProviderAttemptEnvelope.ts b/control-api/src/services/llmProviderAttemptEnvelope.tsnew file mode 100644index 000000000..25b86bb33--- /dev/null+++ b/control-api/src/services/llmProviderAttemptEnvelope.ts@@ -0,0 +1,47 @@+/**+ * How long one Codex or Grok provider attempt can live, and the windows that+ * must outlast it.+ *+ * An attempt starts at authorize: its execution ticket stays redeemable for+ * the ticket TTL, and once redeemed the proxy streams for at most the+ * `maxStreamDurationMs` this service returns on redeem. Spend arrives at+ * finalize, after the stream ends. The budget reservation taken at authorize+ * and the in-flight usage grace the sweepers honour must both cover that+ * whole lifetime plus the rollup lag, or spend lands after the reservation+ * freed its headroom and after the sweeper closed the invocation.+ *+ * The base constants live here, not in the ticket and redemption modules+ * that re-export them: those modules import `llmProviderAttemptStore`, which+ * imports this one, so defining them there would make this module read them+ * before they are initialized whenever the ticket module loads first.+ */+import { config } from '../config.js'++export const CODEX_EXECUTION_TICKET_TTL_SECONDS = 60+export const GROK_EXECUTION_TICKET_TTL_SECONDS = 60++/** Per-attempt stream cap control-api returns on redeem (30 min). */+export const CODEX_MAX_STREAM_DURATION_MS = 1_800_000+export const GROK_MAX_STREAM_DURATION_MS = 1_800_000++export const CODEX_ATTEMPT_MAX_LIFETIME_MS =+ CODEX_EXECUTION_TICKET_TTL_SECONDS * 1000 + CODEX_MAX_STREAM_DURATION_MS+export const GROK_ATTEMPT_MAX_LIFETIME_MS =+ GROK_EXECUTION_TICKET_TTL_SECONDS * 1000 + GROK_MAX_STREAM_DURATION_MS++/**+ * Budget reservation TTL for one attempt: its lifetime plus the task-level+ * reservation TTL, which is already sized to the rollup lag (config.ts).+ */+export const CODEX_ATTEMPT_RESERVATION_TTL_SECONDS =+ Math.ceil(CODEX_ATTEMPT_MAX_LIFETIME_MS / 1000) + config.budgetReservationTtlSeconds+export const GROK_ATTEMPT_RESERVATION_TTL_SECONDS =+ Math.ceil(GROK_ATTEMPT_MAX_LIFETIME_MS / 1000) + config.budgetReservationTtlSeconds++/**+ * How long an authorized or redeemed attempt row without usage still counts+ * as in flight, for either provider.+ */+export const IN_FLIGHT_USAGE_GRACE_MS =+ Math.max(CODEX_ATTEMPT_MAX_LIFETIME_MS, GROK_ATTEMPT_MAX_LIFETIME_MS) ++ config.budgetReservationTtlSeconds * 1000diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.tsindex f0b783e00..6200ebc76 100644--- a/control-api/src/services/llmProviderAttemptRedemption.ts+++ b/control-api/src/services/llmProviderAttemptRedemption.ts@@ -12,6 +12,7 @@ import {CodexSubscriptionOAuthError,ensureFreshCodexAccessToken,} from './codexSubscriptionOAuth.js'+import { CODEX_MAX_STREAM_DURATION_MS } from './llmProviderAttemptEnvelope.js'import {loadLlmProviderAttempt,lockLlmProviderAttemptTicket,@@ -25,7 +26,7 @@ export const CODEX_COMPLETIONS_ORIGIN = 'https://chatgpt.com/backend-api/codex/rexport const CODEX_CATALOG_ORIGIN ='https://chatgpt.com/backend-api/codex/models?client_version=1.0.0'export const CODEX_TRANSPORT_PROTOCOL = 'codex-subscription-transport.v1'-export const CODEX_MAX_STREAM_DURATION_MS = 1_800_000+export { CODEX_MAX_STREAM_DURATION_MS }export type LlmProviderAttemptRedeemErrorCode =| 'disabled'diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.tsindex 4df8573d5..91e341334 100644--- a/control-api/src/services/llmProviderAttemptStore.ts+++ b/control-api/src/services/llmProviderAttemptStore.ts@@ -1,5 +1,6 @@import type { DbClient } from '../db.js'import { rootLogger } from '../observability/logger.js'+import { IN_FLIGHT_USAGE_GRACE_MS } from './llmProviderAttemptEnvelope.js'const log = rootLogger.child({ module: 'llm-provider-attempt-store' })@@ -65,11 +66,14 @@ export function linkedCodexExactUsage(}/**- * How long an authorized/redeemed Codex row may block SDK spend freeze.- * After this, usage is treated as never arriving so the sweeper can close- * the invocation. `finalized` is always terminal: ingest will not add tokens.+ * How long an authorized/redeemed provider attempt row may block SDK spend+ * freeze. It covers both Codex and Grok rows despite the name: the longest+ * attempt lifetime of either provider plus the rollup margin+ * (llmProviderAttemptEnvelope.ts). After this, usage is treated as never+ * arriving so the sweeper can close the invocation. `finalized` is always+ * terminal: ingest will not add tokens.*/-export const CODEX_IN_FLIGHT_USAGE_GRACE_MS = 15 * 60 * 1000+export const CODEX_IN_FLIGHT_USAGE_GRACE_MS = IN_FLIGHT_USAGE_GRACE_MS/** Linked Codex still owns exact usage; sweepers must not freeze SDK spend. */export function isLinkedCodexInFlightWithoutUsage(diff --git a/control-api/src/services/llmProviderAttemptTicket.ts b/control-api/src/services/llmProviderAttemptTicket.tsindex cce22e6e1..28eec5cc9 100644--- a/control-api/src/services/llmProviderAttemptTicket.ts+++ b/control-api/src/services/llmProviderAttemptTicket.ts@@ -2,11 +2,12 @@ import jwt from 'jsonwebtoken'import { createPublicKey, randomUUID } from 'node:crypto'import { config } from '../config.js'import type { DbClient } from '../db.js'+import { CODEX_EXECUTION_TICKET_TTL_SECONDS } from './llmProviderAttemptEnvelope.js'import { registerLlmProviderAttemptTicket } from './llmProviderAttemptStore.js'export const CODEX_EXECUTION_TICKET_TYP = 'codex-execution-ticket' as constexport const CODEX_EXECUTION_TICKET_AUDIENCE = 'codex-llm-proxy'-export const CODEX_EXECUTION_TICKET_TTL_SECONDS = 60+export { CODEX_EXECUTION_TICKET_TTL_SECONDS }export type CodexExecutionTicketClaims = {jti: stringdiff --git a/control-api/test/routes.budgetsCheck.test.ts b/control-api/test/routes.budgetsCheck.test.tsindex d241223ff..d0b44e123 100644--- a/control-api/test/routes.budgetsCheck.test.ts+++ b/control-api/test/routes.budgetsCheck.test.ts@@ -1,7 +1,7 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import request from 'supertest'import { createApp } from '../src/app.js'-import { __resetBudgetCheckCache } from '../src/services/budgets/check.js'+import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'import { sweepExpiredReservations } from '../src/services/budgets/reservations.js'import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'import { MockGateway } from './mockGateway.js'@@ -494,6 +494,29 @@ describe('POST /api/v1/internal/budgets/check — danger-zone reservation', () =)expect(String(insertCall![0])).toMatch(/budget_id, est_amount, task_ref, host_ref, expires_at/)expect((insertCall![1] as unknown[])[3]).toBe('trader')+ // The task-level check keeps the config TTL ($5); only an LLM attempt+ // authorize passes a longer one.+ expect((insertCall![1] as unknown[])[4]).toBe(300)+ })++ it('reserves for the caller-supplied TTL when the check passes one', async () => {+ budgetsRows = [dangerBudget()]+ spendQueue = [{ rows: [{ spent: '950' }] }]++ const result = await evaluateBudgetCheck(+ { ...BASE_REQUEST, task_ref: 'invocation-1:1:1' },+ undefined,+ undefined,+ { reservationTtlSeconds: 2160 }+ )++ expect(result.allowed).toBe(true)+ expect(result.reservationIds).toEqual([reservationId])+ const insertCall = mockTxQuery.mock.calls.find(c =>+ /INSERT INTO budget_pending_reservations/.test(String(c[0]))+ )+ expect(insertCall).toBeDefined()+ expect((insertCall![1] as unknown[])[4]).toBe(2160)})it('excludes the current task_ref from the pending sum (no self double-count)', async () => {diff --git a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.tsindex 93b13e66f..b5891d505 100644--- a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts+++ b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts@@ -3,7 +3,7 @@ import { randomBytes, randomUUID } from 'node:crypto'import { Pool } from 'pg'import { initDb } from '../src/db.js'import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'-import { evaluateBudgetCheck } from '../src/services/budgets/check.js'+import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'import { getActiveReservation } from '../src/services/budgets/reservations.js'import { getCodexCatalogModelState } from '../src/services/codexSubscriptionCatalog.js'import {@@ -16,6 +16,7 @@ import {authorizeLlmProviderAttempt,computeCodexPolicyHash,} from '../src/services/llmProviderAttemptAuthorizer.js'+import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'import {getMaxLlmProviderAttemptGeneration,insertLlmProviderAttempt,@@ -222,4 +223,71 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',)expect(leftover.rows[0]?.count).toBe('0')})++ it('reserves a danger-zone token budget for the whole attempt lifetime', async () => {+ const current = await getSafeCodexSubscriptionConnection(pool)+ // remaining (1000) < max_task_amount (2000) puts the budget in the danger+ // zone, and min_start 0 lets the attempt through with a reservation.+ const budget = await pool.query<{ id: string }>(+ `INSERT INTO token_budgets+ (name, scope, unit, limit_amount, period, timezone,+ min_start_amount, max_task_amount, enforcement)+ VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')+ RETURNING id`+ )+ const budgetId = budget.rows[0]!.id+ __resetBudgetCheckCache()+ try {+ const invocationId = `invocation-envelope-${randomUUID()}`+ const authorized = await authorizeLlmProviderAttempt(+ claims(),+ {+ request: { ...REQUEST, requestId: 'req-pg-3', idempotencyKey: 'idem-pg-3' },+ invocationId,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ policyRevision: current!.catalogRevision,+ policyHash: computeCodexPolicyHash({+ model: REQUEST.model,+ catalogRevision: current!.catalogRevision,+ credentialRevision: current!.credentialRevision,+ connectionKey: current!.connectionKey,+ }),+ },+ testDeps()+ )+ expect(authorized.providerAttemptId).toEqual(expect.any(String))++ // created_at and expires_at come from the same statement's NOW(), so+ // their difference is exactly the TTL the reservation was written with.+ const reservations = await pool.query<{+ id: string+ task_ref: string+ ttl_seconds: number+ }>(+ `SELECT id, task_ref,+ EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds+ FROM budget_pending_reservations+ WHERE budget_id = $1`,+ [budgetId]+ )+ expect(reservations.rows).toEqual([+ {+ id: expect.any(String),+ task_ref: `${invocationId}:1:1`,+ ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ },+ ])+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)++ const attempt = await pool.query<{ budget_reservation_id: string }>(+ `SELECT budget_reservation_id FROM llm_provider_attempts WHERE invocation_id = $1`,+ [invocationId]+ )+ expect(attempt.rows).toEqual([{ budget_reservation_id: reservations.rows[0]!.id }])+ } finally {+ await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])+ __resetBudgetCheckCache()+ }+ })})diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.tsindex 1b4688158..ee3ff03c6 100644--- a/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts+++ b/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts@@ -260,6 +260,12 @@ describe('authorizeLlmProviderAttempt grok-subscription', () => {requestHash: hashGrokCompletionRequestV1(REQUEST),})expect(grokRepos.getConnection).toHaveBeenCalledWith(expect.anything(), 'team-grok')+ expect(current.evaluateBudget).toHaveBeenCalledWith(+ expect.objectContaining({ provider: 'grok-subscription' }),+ expect.anything(),+ expect.anything(),+ expect.objectContaining({ requiredUnit: 'tokens', reservationTtlSeconds: 2160 })+ )expect(current.insertAttempt).toHaveBeenCalledWith(expect.anything(),expect.objectContaining({diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.test.tsindex bf5a2f531..b9adcdad9 100644--- a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts+++ b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts@@ -180,7 +180,7 @@ describe('authorizeLlmProviderAttempt', () => {}),expect.anything(),expect.anything(),- expect.objectContaining({ requiredUnit: 'tokens' })+ expect.objectContaining({ requiredUnit: 'tokens', reservationTtlSeconds: 2160 }))expect(current.insertAttempt).toHaveBeenCalledWith(expect.anything(),diff --git a/control-api/test/services.llmProviderAttemptEnvelope.test.ts b/control-api/test/services.llmProviderAttemptEnvelope.test.tsnew file mode 100644index 000000000..d0a883882--- /dev/null+++ b/control-api/test/services.llmProviderAttemptEnvelope.test.ts@@ -0,0 +1,51 @@+import { describe, expect, it } from 'vitest'+import { config } from '../src/config.js'+import { GROK_MAX_STREAM_DURATION_MS } from '../src/services/grokProviderAttemptRedemption.js'+import { GROK_EXECUTION_TICKET_TTL_SECONDS } from '../src/services/grokProviderAttemptTicket.js'+import {+ CODEX_ATTEMPT_MAX_LIFETIME_MS,+ CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ GROK_ATTEMPT_MAX_LIFETIME_MS,+ GROK_ATTEMPT_RESERVATION_TTL_SECONDS,+ IN_FLIGHT_USAGE_GRACE_MS,+} from '../src/services/llmProviderAttemptEnvelope.js'+import { CODEX_MAX_STREAM_DURATION_MS } from '../src/services/llmProviderAttemptRedemption.js'+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'+import { CODEX_EXECUTION_TICKET_TTL_SECONDS } from '../src/services/llmProviderAttemptTicket.js'++// One provider attempt lives from authorize until its stream ends: the ticket+// can wait its TTL before the redeem, then the stream runs up to the redeem+// cap. The budget reservation and the in-flight usage grace must both outlast+// that, or spend arrives after the reservation freed the budget headroom.+describe('provider attempt lifetime envelope', () => {+ it('derives each attempt lifetime from the ticket TTL and the redeem stream cap', () => {+ expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(+ CODEX_EXECUTION_TICKET_TTL_SECONDS * 1000 + CODEX_MAX_STREAM_DURATION_MS+ )+ expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(+ GROK_EXECUTION_TICKET_TTL_SECONDS * 1000 + GROK_MAX_STREAM_DURATION_MS+ )+ expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)+ expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)+ })++ it('keeps each attempt reservation alive for the attempt plus the rollup margin', () => {+ const margin = config.budgetReservationTtlSeconds+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(+ Math.ceil(CODEX_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin+ )+ expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(+ Math.ceil(GROK_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin+ )+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)+ expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)+ })++ it('holds the in-flight usage grace past the longest attempt of either provider', () => {+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBeGreaterThan(CODEX_ATTEMPT_MAX_LIFETIME_MS)+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBeGreaterThan(GROK_ATTEMPT_MAX_LIFETIME_MS)+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBe(2_160_000)+ // The sweepers read the Store constant; it must be this derived value.+ expect(CODEX_IN_FLIGHT_USAGE_GRACE_MS).toBe(IN_FLIGHT_USAGE_GRACE_MS)+ })+})diff --git a/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.tsindex 59e3a6bc3..481a82d32 100644--- a/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts+++ b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts@@ -4,10 +4,12 @@ import { Pool } from 'pg'import { config } from '../src/config.js'import { initDb } from '../src/db.js'import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'+import { reserveInDangerZone } from '../src/services/budgets/reservations.js'import {insertInitialCodexSubscriptionConnection,loadCodexSubscriptionSecrets,} from '../src/services/codexSubscriptionConnection.js'+import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'import { finalizeLlmProviderAttempt } from '../src/services/llmProviderAttemptFinalization.js'import { opaqueAttemptReceipt } from '../src/services/llmProviderAttemptRedemption.js'import { redeemLlmProviderAttempt } from '../src/services/llmProviderAttemptRedemption.js'@@ -273,4 +275,102 @@ describeRealPostgres('Codex attempt finalization on real PostgreSQL', () => {provider: 'codex-subscription',})})++ it('releases the attempt-lifetime budget reservation when the attempt finalizes', async () => {+ const budget = await pool.query<{ id: string }>(+ `INSERT INTO token_budgets+ (name, scope, unit, limit_amount, period, timezone,+ min_start_amount, max_task_amount, enforcement)+ VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')+ RETURNING id`+ )+ const budgetId = budget.rows[0]!.id+ try {+ const invocationId = `invocation-${randomUUID()}`+ const reserved = await reserveInDangerZone(+ {+ budgetId,+ limit: 1000,+ spent: 0,+ minStart: 0,+ estAmount: 2000,+ taskRef: `${invocationId}:1:1`,+ hostRef: 'research-host',+ ttlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,+ },+ pool+ )+ if (reserved.decision !== 'allow') throw new Error('expected the reservation to be allowed')++ const attempt = await insertLlmProviderAttempt(pool, {+ callerKind: 'host',+ hostRef: 'research-host',+ invocationId,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ model: 'gpt-5.1',+ requestHash: 'b'.repeat(64),+ policyRevision: 1,+ policyHash: 'a'.repeat(64),+ budgetReservationId: reserved.reservationId,+ connectionRevision: 1,+ connectionId,+ })+ const issued = await issueRegisteredCodexExecutionTicket(pool, {+ sub: 'host/research-host',+ hostRef: attempt.hostRef,+ invocationId: attempt.invocationId,+ attemptGeneration: attempt.attemptGeneration,+ providerAttemptId: attempt.id,+ providerAttemptIndex: attempt.providerAttemptIndex,+ model: attempt.model,+ requestHash: attempt.requestHash,+ policyRevision: attempt.policyRevision,+ policyHash: attempt.policyHash,+ budgetReservationId: attempt.budgetReservationId,+ connectionRevision: attempt.connectionRevision,+ })+ await redeemLlmProviderAttempt(+ { executionTicket: issued.executionTicket, requestHash: attempt.requestHash },+ txDeps()+ )++ // Witness: the reservation is live, with the attempt-lifetime TTL, right+ // before finalize, so its absence afterwards is the release.+ const before = await pool.query<{ ttl_seconds: number }>(+ `SELECT EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds+ FROM budget_pending_reservations+ WHERE id = $1 AND expires_at > NOW()`,+ [reserved.reservationId]+ )+ expect(before.rows).toEqual([{ ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS }])++ const finalized = await finalizeLlmProviderAttempt(+ {+ attemptReceipt: opaqueAttemptReceipt({+ jti: issued.claims.jti,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ }),+ receipt: {+ schemaVersion: 'codex-attempt-receipt.v1' as const,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ outcome: 'success' as const,+ usage: { inputTokens: 7, outputTokens: 3 },+ },+ },+ runTx+ )+ expect(finalized).toMatchObject({ outcome: 'success', duplicate: false })++ const after = await pool.query<{ count: number }>(+ `SELECT COUNT(*)::int AS count FROM budget_pending_reservations WHERE id = $1`,+ [reserved.reservationId]+ )+ expect(after.rows[0]?.count).toBe(0)+ } finally {+ await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])+ }+ })})diff --git a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts b/control-api/test/services.llmProviderAttemptStore.helpers.test.tsindex a338acfc6..db5f4b37f 100644--- a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts+++ b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts@@ -1,4 +1,8 @@import { describe, expect, it } from 'vitest'+import {+ CODEX_ATTEMPT_MAX_LIFETIME_MS,+ GROK_ATTEMPT_MAX_LIFETIME_MS,+} from '../src/services/llmProviderAttemptEnvelope.js'import {CODEX_IN_FLIGHT_USAGE_GRACE_MS,isLinkedCodexInFlightWithoutUsage,@@ -100,6 +104,25 @@ describe('linked Codex usage predicates', () => {).toBe(false)})+ it('keeps a redeemed row in flight for the whole attempt lifetime of either provider', () => {+ // A 30 min stream that authorized a minute before its redeem is still+ // billing; sweeping it would close the invocation under a live stream.+ for (const lifetime of [CODEX_ATTEMPT_MAX_LIFETIME_MS, GROK_ATTEMPT_MAX_LIFETIME_MS]) {+ expect(+ isLinkedCodexInFlightWithoutUsage(+ {+ status: 'redeemed',+ outcome: null,+ usageInputTokens: null,+ usageOutputTokens: null,+ createdAt: new Date(nowMs - lifetime),+ },+ nowMs+ )+ ).toBe(true)+ }+ })+it('ages out authorized and redeemed rows after the usage grace', () => {const stale = new Date(nowMs - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 1)expect(diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.tsindex a82c84454..f2c6ea17a 100644--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts@@ -1,5 +1,6 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import { pool } from '../src/db.js'+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'import {PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,buildGrantUpdateNotifyPayload,@@ -583,7 +584,7 @@ describe('invocation retry lifecycle timestamps', () => {outcome: null,usage_input_tokens: null,usage_output_tokens: null,- created_at: new Date(Date.now() - 16 * 60 * 1000),+ created_at: new Date(Date.now() - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 60_000),},],rowCount: 1,diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.tsindex 2babc2f9a..82d293d6b 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts@@ -2,7 +2,10 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'import { randomBytes, randomUUID } from 'node:crypto'import { Pool, type PoolClient } from 'pg'import { initDb } from '../src/db.js'-import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'+import {+ CODEX_IN_FLIGHT_USAGE_GRACE_MS,+ insertLlmProviderAttempt,+} from '../src/services/llmProviderAttemptStore.js'import { failStaleInvocationsInTransaction } from '../src/services/pluginWorkloadSdkDb.js'import {PromptBridgeFinalizationError,@@ -597,8 +600,10 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =// Out of the in-flight grace, so the sweeper is free to close it.const codexAttemptId = await linkCodex(client, input)await client.query(- `UPDATE llm_provider_attempts SET created_at = now() - interval '30 minutes' WHERE id = $1`,- [codexAttemptId]+ `UPDATE llm_provider_attempts+ SET created_at = now() - make_interval(secs => $2::double precision / 1000)+ WHERE id = $1`,+ [codexAttemptId, CODEX_IN_FLIGHT_USAGE_GRACE_MS + 60_000])expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)await commit(client)diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.test.tsindex a342bce38..15d104907 100644--- a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts+++ b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts@@ -1,4 +1,5 @@import { describe, expect, it, vi } from 'vitest'+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'import {PromptBridgeFinalizationError,deriveEffectiveSpend,@@ -1402,7 +1403,10 @@ describe('spend floor writers (Addendum A.4 "best-floor")', () => {[],[],...codexFence('in_progress'),- linkedCodexRow({ ...codexInFlight, created_at: new Date(Date.now() - 20 * 60_000) }),+ linkedCodexRow({+ ...codexInFlight,+ created_at: new Date(Date.now() - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 60_000),+ }),{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },{ rows: [], rowCount: 1 },