Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): hold budget reservations and usage grace for the whole attempt

Alfredo Lopez8ed35d13983b
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index f64de5231..716800e60 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 20033d94b..a4c6f4544 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.600",
+ "version": "0.1.601",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/config.ts b/control-api/src/config.ts
index 41ddda9ca..fa82b204e 100644
--- a/control-api/src/config.ts
+++ b/control-api/src/config.ts
@@ -909,6 +909,9 @@ export const config: Config = {
// Danger-zone reservation TTL (§9.8a: ~2-3× the rollup lag, ~5 min). Short
// enough that a hung reservation auto-frees; long enough that real spend has
// reached the rollups before it expires (no double-count on the next check).
+ // This is the task-level TTL. A Codex or Grok provider attempt reserves for
+ // its whole lifetime plus this value, and the in-flight usage grace adds it
+ // to the longest attempt (services/llmProviderAttemptEnvelope.ts).
budgetReservationTtlSeconds: positiveIntegerFromEnv('BUDGET_RESERVATION_TTL_SECONDS', 300),
// Default 180 days. Archival runs daily at 02:00 UTC; older terminal
// approvals move to the archive table.
diff --git a/control-api/src/services/budgets/check.ts b/control-api/src/services/budgets/check.ts
index cd15359e1..63313e341 100644
--- a/control-api/src/services/budgets/check.ts
+++ b/control-api/src/services/budgets/check.ts
@@ -100,6 +100,8 @@ export type BudgetCheckOptions = {
requiredUnit?: 'tokens'
/** When set, danger-zone reservations join this already-open transaction. */
transactionClient?: ReservationTxClient
+ /** Danger-zone reservation TTL in seconds; absent keeps the config TTL. */
+ reservationTtlSeconds?: number
}
export type BudgetCheckResult = {
@@ -413,6 +415,7 @@ export async function evaluateBudgetCheck(
// free this host's own reservations (§5.4). Already normalized
// above and bound to claims.hostRefs[0] by the route's claim-binding.
hostRef: dimensions.host_ref ?? null,
+ ttlSeconds: options.reservationTtlSeconds,
},
connector,
options.transactionClient
diff --git a/control-api/src/services/budgets/reservations.ts b/control-api/src/services/budgets/reservations.ts
index 3155a879c..2501a2ab1 100644
--- a/control-api/src/services/budgets/reservations.ts
+++ b/control-api/src/services/budgets/reservations.ts
@@ -66,6 +66,12 @@ export type DangerZoneReserveInput = {
* by the TTL sweep, never by an early release.
*/
hostRef: string | null
+ /**
+ * Reservation lifetime in seconds. Absent for a task-level check, which
+ * keeps BUDGET_RESERVATION_TTL_SECONDS; an LLM provider attempt passes its
+ * own, because its spend arrives only when a stream of up to 30 min ends.
+ */
+ ttlSeconds?: number
}
export type DangerZoneReserveResult =
@@ -145,7 +151,13 @@ async function decideDangerZoneReservation(
`INSERT INTO budget_pending_reservations (budget_id, est_amount, task_ref, host_ref, expires_at)
VALUES ($1, $2, $3, $4, NOW() + ($5::int * INTERVAL '1 second'))
RETURNING id`,
- [input.budgetId, input.estAmount, input.taskRef, input.hostRef, BUDGET_RESERVATION_TTL_SECONDS]
+ [
+ input.budgetId,
+ input.estAmount,
+ input.taskRef,
+ input.hostRef,
+ input.ttlSeconds ?? BUDGET_RESERVATION_TTL_SECONDS,
+ ]
)
const reservationId = String((insertRes.rows[0] as { id: unknown }).id)
return { decision: 'allow', reservationId }
diff --git a/control-api/src/services/grokProviderAttemptRedemption.ts b/control-api/src/services/grokProviderAttemptRedemption.ts
index 8c759a278..364dfe432 100644
--- a/control-api/src/services/grokProviderAttemptRedemption.ts
+++ b/control-api/src/services/grokProviderAttemptRedemption.ts
@@ -13,6 +13,7 @@ import {
loadGrokSubscriptionSecrets,
} from './grokSubscriptionConnection.js'
import { GrokSubscriptionOAuthError, ensureFreshGrokAccessToken } from './grokSubscriptionOAuth.js'
+import { GROK_MAX_STREAM_DURATION_MS } from './llmProviderAttemptEnvelope.js'
import { opaqueAttemptReceipt } from './llmProviderAttemptRedemption.js'
import {
loadLlmProviderAttempt,
@@ -26,7 +27,7 @@ const log = rootLogger.child({ module: 'grok-provider-attempt-redemption' })
export const GROK_COMPLETIONS_ORIGIN = COMPLETIONS_ORIGIN
export const GROK_CATALOG_ORIGIN = CATALOG_ORIGIN
export const GROK_TRANSPORT_PROTOCOL = TRANSPORT_PROTOCOL_VERSION
-export const GROK_MAX_STREAM_DURATION_MS = 1_800_000
+export { GROK_MAX_STREAM_DURATION_MS }
export type GrokProviderAttemptRedeemErrorCode =
| 'disabled'
diff --git a/control-api/src/services/grokProviderAttemptTicket.ts b/control-api/src/services/grokProviderAttemptTicket.ts
index 9bcc1ac0c..ae36a4606 100644
--- a/control-api/src/services/grokProviderAttemptTicket.ts
+++ b/control-api/src/services/grokProviderAttemptTicket.ts
@@ -2,11 +2,12 @@ import jwt from 'jsonwebtoken'
import { createPublicKey, randomUUID } from 'node:crypto'
import { config } from '../config.js'
import type { DbClient } from '../db.js'
+import { GROK_EXECUTION_TICKET_TTL_SECONDS } from './llmProviderAttemptEnvelope.js'
import { registerLlmProviderAttemptTicket } from './llmProviderAttemptStore.js'
export const GROK_EXECUTION_TICKET_TYP = 'grok-execution-ticket' as const
export const GROK_EXECUTION_TICKET_AUDIENCE = 'grok-llm-proxy'
-export const GROK_EXECUTION_TICKET_TTL_SECONDS = 60
+export { GROK_EXECUTION_TICKET_TTL_SECONDS }
export type GrokExecutionTicketClaims = {
jti: string
diff --git a/control-api/src/services/llmProviderAttemptAuthorizer.ts b/control-api/src/services/llmProviderAttemptAuthorizer.ts
index e428f1a88..051b0fcc6 100644
--- a/control-api/src/services/llmProviderAttemptAuthorizer.ts
+++ b/control-api/src/services/llmProviderAttemptAuthorizer.ts
@@ -31,6 +31,10 @@ import {
getSafeGrokSubscriptionConnection,
isGrokUnassignedConnectionKey,
} from './grokSubscriptionConnection.js'
+import {
+ CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ GROK_ATTEMPT_RESERVATION_TTL_SECONDS,
+} from './llmProviderAttemptEnvelope.js'
import {
getMaxLlmProviderAttemptGeneration,
insertLlmProviderAttempt,
@@ -452,7 +456,11 @@ async function authorizeGrokProviderAttempt(
},
db,
{ connect: async () => tx as never },
- { requiredUnit: 'tokens', transactionClient: tx }
+ {
+ requiredUnit: 'tokens',
+ transactionClient: tx,
+ reservationTtlSeconds: GROK_ATTEMPT_RESERVATION_TTL_SECONDS,
+ }
)
if (!budget.allowed) {
throw new LlmProviderAttemptAuthorizeError(
@@ -835,7 +843,11 @@ export async function authorizeLlmProviderAttempt(
},
db,
{ connect: async () => tx as never },
- { requiredUnit: 'tokens', transactionClient: tx }
+ {
+ requiredUnit: 'tokens',
+ transactionClient: tx,
+ reservationTtlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ }
)
if (!budget.allowed) {
throw new LlmProviderAttemptAuthorizeError(
diff --git a/control-api/src/services/llmProviderAttemptEnvelope.ts b/control-api/src/services/llmProviderAttemptEnvelope.ts
new file mode 100644
index 000000000..25b86bb33
--- /dev/null
+++ b/control-api/src/services/llmProviderAttemptEnvelope.ts
@@ -0,0 +1,47 @@
+/**
+ * How long one Codex or Grok provider attempt can live, and the windows that
+ * must outlast it.
+ *
+ * An attempt starts at authorize: its execution ticket stays redeemable for
+ * the ticket TTL, and once redeemed the proxy streams for at most the
+ * `maxStreamDurationMs` this service returns on redeem. Spend arrives at
+ * finalize, after the stream ends. The budget reservation taken at authorize
+ * and the in-flight usage grace the sweepers honour must both cover that
+ * whole lifetime plus the rollup lag, or spend lands after the reservation
+ * freed its headroom and after the sweeper closed the invocation.
+ *
+ * The base constants live here, not in the ticket and redemption modules
+ * that re-export them: those modules import `llmProviderAttemptStore`, which
+ * imports this one, so defining them there would make this module read them
+ * before they are initialized whenever the ticket module loads first.
+ */
+import { config } from '../config.js'
+
+export const CODEX_EXECUTION_TICKET_TTL_SECONDS = 60
+export const GROK_EXECUTION_TICKET_TTL_SECONDS = 60
+
+/** Per-attempt stream cap control-api returns on redeem (30 min). */
+export const CODEX_MAX_STREAM_DURATION_MS = 1_800_000
+export const GROK_MAX_STREAM_DURATION_MS = 1_800_000
+
+export const CODEX_ATTEMPT_MAX_LIFETIME_MS =
+ CODEX_EXECUTION_TICKET_TTL_SECONDS * 1000 + CODEX_MAX_STREAM_DURATION_MS
+export const GROK_ATTEMPT_MAX_LIFETIME_MS =
+ GROK_EXECUTION_TICKET_TTL_SECONDS * 1000 + GROK_MAX_STREAM_DURATION_MS
+
+/**
+ * Budget reservation TTL for one attempt: its lifetime plus the task-level
+ * reservation TTL, which is already sized to the rollup lag (config.ts).
+ */
+export const CODEX_ATTEMPT_RESERVATION_TTL_SECONDS =
+ Math.ceil(CODEX_ATTEMPT_MAX_LIFETIME_MS / 1000) + config.budgetReservationTtlSeconds
+export const GROK_ATTEMPT_RESERVATION_TTL_SECONDS =
+ Math.ceil(GROK_ATTEMPT_MAX_LIFETIME_MS / 1000) + config.budgetReservationTtlSeconds
+
+/**
+ * How long an authorized or redeemed attempt row without usage still counts
+ * as in flight, for either provider.
+ */
+export const IN_FLIGHT_USAGE_GRACE_MS =
+ Math.max(CODEX_ATTEMPT_MAX_LIFETIME_MS, GROK_ATTEMPT_MAX_LIFETIME_MS) +
+ config.budgetReservationTtlSeconds * 1000
diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.ts
index f0b783e00..6200ebc76 100644
--- a/control-api/src/services/llmProviderAttemptRedemption.ts
+++ b/control-api/src/services/llmProviderAttemptRedemption.ts
@@ -12,6 +12,7 @@ import {
CodexSubscriptionOAuthError,
ensureFreshCodexAccessToken,
} from './codexSubscriptionOAuth.js'
+import { CODEX_MAX_STREAM_DURATION_MS } from './llmProviderAttemptEnvelope.js'
import {
loadLlmProviderAttempt,
lockLlmProviderAttemptTicket,
@@ -25,7 +26,7 @@ export const CODEX_COMPLETIONS_ORIGIN = 'https://chatgpt.com/backend-api/codex/r
export const CODEX_CATALOG_ORIGIN =
'https://chatgpt.com/backend-api/codex/models?client_version=1.0.0'
export const CODEX_TRANSPORT_PROTOCOL = 'codex-subscription-transport.v1'
-export const CODEX_MAX_STREAM_DURATION_MS = 1_800_000
+export { CODEX_MAX_STREAM_DURATION_MS }
export type LlmProviderAttemptRedeemErrorCode =
| 'disabled'
diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.ts
index 4df8573d5..91e341334 100644
--- a/control-api/src/services/llmProviderAttemptStore.ts
+++ b/control-api/src/services/llmProviderAttemptStore.ts
@@ -1,5 +1,6 @@
import type { DbClient } from '../db.js'
import { rootLogger } from '../observability/logger.js'
+import { IN_FLIGHT_USAGE_GRACE_MS } from './llmProviderAttemptEnvelope.js'
const log = rootLogger.child({ module: 'llm-provider-attempt-store' })
@@ -65,11 +66,14 @@ export function linkedCodexExactUsage(
}
/**
- * How long an authorized/redeemed Codex row may block SDK spend freeze.
- * After this, usage is treated as never arriving so the sweeper can close
- * the invocation. `finalized` is always terminal: ingest will not add tokens.
+ * How long an authorized/redeemed provider attempt row may block SDK spend
+ * freeze. It covers both Codex and Grok rows despite the name: the longest
+ * attempt lifetime of either provider plus the rollup margin
+ * (llmProviderAttemptEnvelope.ts). After this, usage is treated as never
+ * arriving so the sweeper can close the invocation. `finalized` is always
+ * terminal: ingest will not add tokens.
*/
-export const CODEX_IN_FLIGHT_USAGE_GRACE_MS = 15 * 60 * 1000
+export const CODEX_IN_FLIGHT_USAGE_GRACE_MS = IN_FLIGHT_USAGE_GRACE_MS
/** Linked Codex still owns exact usage; sweepers must not freeze SDK spend. */
export function isLinkedCodexInFlightWithoutUsage(
diff --git a/control-api/src/services/llmProviderAttemptTicket.ts b/control-api/src/services/llmProviderAttemptTicket.ts
index cce22e6e1..28eec5cc9 100644
--- a/control-api/src/services/llmProviderAttemptTicket.ts
+++ b/control-api/src/services/llmProviderAttemptTicket.ts
@@ -2,11 +2,12 @@ import jwt from 'jsonwebtoken'
import { createPublicKey, randomUUID } from 'node:crypto'
import { config } from '../config.js'
import type { DbClient } from '../db.js'
+import { CODEX_EXECUTION_TICKET_TTL_SECONDS } from './llmProviderAttemptEnvelope.js'
import { registerLlmProviderAttemptTicket } from './llmProviderAttemptStore.js'
export const CODEX_EXECUTION_TICKET_TYP = 'codex-execution-ticket' as const
export const CODEX_EXECUTION_TICKET_AUDIENCE = 'codex-llm-proxy'
-export const CODEX_EXECUTION_TICKET_TTL_SECONDS = 60
+export { CODEX_EXECUTION_TICKET_TTL_SECONDS }
export type CodexExecutionTicketClaims = {
jti: string
diff --git a/control-api/test/routes.budgetsCheck.test.ts b/control-api/test/routes.budgetsCheck.test.ts
index d241223ff..d0b44e123 100644
--- a/control-api/test/routes.budgetsCheck.test.ts
+++ b/control-api/test/routes.budgetsCheck.test.ts
@@ -1,7 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import request from 'supertest'
import { createApp } from '../src/app.js'
-import { __resetBudgetCheckCache } from '../src/services/budgets/check.js'
+import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { sweepExpiredReservations } from '../src/services/budgets/reservations.js'
import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'
import { MockGateway } from './mockGateway.js'
@@ -494,6 +494,29 @@ describe('POST /api/v1/internal/budgets/check — danger-zone reservation', () =
)
expect(String(insertCall![0])).toMatch(/budget_id, est_amount, task_ref, host_ref, expires_at/)
expect((insertCall![1] as unknown[])[3]).toBe('trader')
+ // The task-level check keeps the config TTL ($5); only an LLM attempt
+ // authorize passes a longer one.
+ expect((insertCall![1] as unknown[])[4]).toBe(300)
+ })
+
+ it('reserves for the caller-supplied TTL when the check passes one', async () => {
+ budgetsRows = [dangerBudget()]
+ spendQueue = [{ rows: [{ spent: '950' }] }]
+
+ const result = await evaluateBudgetCheck(
+ { ...BASE_REQUEST, task_ref: 'invocation-1:1:1' },
+ undefined,
+ undefined,
+ { reservationTtlSeconds: 2160 }
+ )
+
+ expect(result.allowed).toBe(true)
+ expect(result.reservationIds).toEqual([reservationId])
+ const insertCall = mockTxQuery.mock.calls.find(c =>
+ /INSERT INTO budget_pending_reservations/.test(String(c[0]))
+ )
+ expect(insertCall).toBeDefined()
+ expect((insertCall![1] as unknown[])[4]).toBe(2160)
})
it('excludes the current task_ref from the pending sum (no self double-count)', async () => {
diff --git a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
index 93b13e66f..b5891d505 100644
--- a/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts
@@ -3,7 +3,7 @@ import { randomBytes, randomUUID } from 'node:crypto'
import { Pool } from 'pg'
import { initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
-import { evaluateBudgetCheck } from '../src/services/budgets/check.js'
+import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { getActiveReservation } from '../src/services/budgets/reservations.js'
import { getCodexCatalogModelState } from '../src/services/codexSubscriptionCatalog.js'
import {
@@ -16,6 +16,7 @@ import {
authorizeLlmProviderAttempt,
computeCodexPolicyHash,
} from '../src/services/llmProviderAttemptAuthorizer.js'
+import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'
import {
getMaxLlmProviderAttemptGeneration,
insertLlmProviderAttempt,
@@ -222,4 +223,71 @@ describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL',
)
expect(leftover.rows[0]?.count).toBe('0')
})
+
+ it('reserves a danger-zone token budget for the whole attempt lifetime', async () => {
+ const current = await getSafeCodexSubscriptionConnection(pool)
+ // remaining (1000) < max_task_amount (2000) puts the budget in the danger
+ // zone, and min_start 0 lets the attempt through with a reservation.
+ const budget = await pool.query<{ id: string }>(
+ `INSERT INTO token_budgets
+ (name, scope, unit, limit_amount, period, timezone,
+ min_start_amount, max_task_amount, enforcement)
+ VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')
+ RETURNING id`
+ )
+ const budgetId = budget.rows[0]!.id
+ __resetBudgetCheckCache()
+ try {
+ const invocationId = `invocation-envelope-${randomUUID()}`
+ const authorized = await authorizeLlmProviderAttempt(
+ claims(),
+ {
+ request: { ...REQUEST, requestId: 'req-pg-3', idempotencyKey: 'idem-pg-3' },
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ policyRevision: current!.catalogRevision,
+ policyHash: computeCodexPolicyHash({
+ model: REQUEST.model,
+ catalogRevision: current!.catalogRevision,
+ credentialRevision: current!.credentialRevision,
+ connectionKey: current!.connectionKey,
+ }),
+ },
+ testDeps()
+ )
+ expect(authorized.providerAttemptId).toEqual(expect.any(String))
+
+ // created_at and expires_at come from the same statement's NOW(), so
+ // their difference is exactly the TTL the reservation was written with.
+ const reservations = await pool.query<{
+ id: string
+ task_ref: string
+ ttl_seconds: number
+ }>(
+ `SELECT id, task_ref,
+ EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds
+ FROM budget_pending_reservations
+ WHERE budget_id = $1`,
+ [budgetId]
+ )
+ expect(reservations.rows).toEqual([
+ {
+ id: expect.any(String),
+ task_ref: `${invocationId}:1:1`,
+ ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ },
+ ])
+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
+
+ const attempt = await pool.query<{ budget_reservation_id: string }>(
+ `SELECT budget_reservation_id FROM llm_provider_attempts WHERE invocation_id = $1`,
+ [invocationId]
+ )
+ expect(attempt.rows).toEqual([{ budget_reservation_id: reservations.rows[0]!.id }])
+ } finally {
+ await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
+ __resetBudgetCheckCache()
+ }
+ })
})
diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts
index 1b4688158..ee3ff03c6 100644
--- a/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorizer.grok.test.ts
@@ -260,6 +260,12 @@ describe('authorizeLlmProviderAttempt grok-subscription', () => {
requestHash: hashGrokCompletionRequestV1(REQUEST),
})
expect(grokRepos.getConnection).toHaveBeenCalledWith(expect.anything(), 'team-grok')
+ expect(current.evaluateBudget).toHaveBeenCalledWith(
+ expect.objectContaining({ provider: 'grok-subscription' }),
+ expect.anything(),
+ expect.anything(),
+ expect.objectContaining({ requiredUnit: 'tokens', reservationTtlSeconds: 2160 })
+ )
expect(current.insertAttempt).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({
diff --git a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
index bf5a2f531..b9adcdad9 100644
--- a/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
+++ b/control-api/test/services.llmProviderAttemptAuthorizer.test.ts
@@ -180,7 +180,7 @@ describe('authorizeLlmProviderAttempt', () => {
}),
expect.anything(),
expect.anything(),
- expect.objectContaining({ requiredUnit: 'tokens' })
+ expect.objectContaining({ requiredUnit: 'tokens', reservationTtlSeconds: 2160 })
)
expect(current.insertAttempt).toHaveBeenCalledWith(
expect.anything(),
diff --git a/control-api/test/services.llmProviderAttemptEnvelope.test.ts b/control-api/test/services.llmProviderAttemptEnvelope.test.ts
new file mode 100644
index 000000000..d0a883882
--- /dev/null
+++ b/control-api/test/services.llmProviderAttemptEnvelope.test.ts
@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+import { config } from '../src/config.js'
+import { GROK_MAX_STREAM_DURATION_MS } from '../src/services/grokProviderAttemptRedemption.js'
+import { GROK_EXECUTION_TICKET_TTL_SECONDS } from '../src/services/grokProviderAttemptTicket.js'
+import {
+ CODEX_ATTEMPT_MAX_LIFETIME_MS,
+ CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ GROK_ATTEMPT_MAX_LIFETIME_MS,
+ GROK_ATTEMPT_RESERVATION_TTL_SECONDS,
+ IN_FLIGHT_USAGE_GRACE_MS,
+} from '../src/services/llmProviderAttemptEnvelope.js'
+import { CODEX_MAX_STREAM_DURATION_MS } from '../src/services/llmProviderAttemptRedemption.js'
+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'
+import { CODEX_EXECUTION_TICKET_TTL_SECONDS } from '../src/services/llmProviderAttemptTicket.js'
+
+// One provider attempt lives from authorize until its stream ends: the ticket
+// can wait its TTL before the redeem, then the stream runs up to the redeem
+// cap. The budget reservation and the in-flight usage grace must both outlast
+// that, or spend arrives after the reservation freed the budget headroom.
+describe('provider attempt lifetime envelope', () => {
+ it('derives each attempt lifetime from the ticket TTL and the redeem stream cap', () => {
+ expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(
+ CODEX_EXECUTION_TICKET_TTL_SECONDS * 1000 + CODEX_MAX_STREAM_DURATION_MS
+ )
+ expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(
+ GROK_EXECUTION_TICKET_TTL_SECONDS * 1000 + GROK_MAX_STREAM_DURATION_MS
+ )
+ expect(CODEX_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)
+ expect(GROK_ATTEMPT_MAX_LIFETIME_MS).toBe(1_860_000)
+ })
+
+ it('keeps each attempt reservation alive for the attempt plus the rollup margin', () => {
+ const margin = config.budgetReservationTtlSeconds
+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(
+ Math.ceil(CODEX_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin
+ )
+ expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(
+ Math.ceil(GROK_ATTEMPT_MAX_LIFETIME_MS / 1000) + margin
+ )
+ expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
+ expect(GROK_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
+ })
+
+ it('holds the in-flight usage grace past the longest attempt of either provider', () => {
+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBeGreaterThan(CODEX_ATTEMPT_MAX_LIFETIME_MS)
+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBeGreaterThan(GROK_ATTEMPT_MAX_LIFETIME_MS)
+ expect(IN_FLIGHT_USAGE_GRACE_MS).toBe(2_160_000)
+ // The sweepers read the Store constant; it must be this derived value.
+ expect(CODEX_IN_FLIGHT_USAGE_GRACE_MS).toBe(IN_FLIGHT_USAGE_GRACE_MS)
+ })
+})
diff --git a/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts
index 59e3a6bc3..481a82d32 100644
--- a/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts
+++ b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts
@@ -4,10 +4,12 @@ import { Pool } from 'pg'
import { config } from '../src/config.js'
import { initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
+import { reserveInDangerZone } from '../src/services/budgets/reservations.js'
import {
insertInitialCodexSubscriptionConnection,
loadCodexSubscriptionSecrets,
} from '../src/services/codexSubscriptionConnection.js'
+import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'
import { finalizeLlmProviderAttempt } from '../src/services/llmProviderAttemptFinalization.js'
import { opaqueAttemptReceipt } from '../src/services/llmProviderAttemptRedemption.js'
import { redeemLlmProviderAttempt } from '../src/services/llmProviderAttemptRedemption.js'
@@ -273,4 +275,102 @@ describeRealPostgres('Codex attempt finalization on real PostgreSQL', () => {
provider: 'codex-subscription',
})
})
+
+ it('releases the attempt-lifetime budget reservation when the attempt finalizes', async () => {
+ const budget = await pool.query<{ id: string }>(
+ `INSERT INTO token_budgets
+ (name, scope, unit, limit_amount, period, timezone,
+ min_start_amount, max_task_amount, enforcement)
+ VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')
+ RETURNING id`
+ )
+ const budgetId = budget.rows[0]!.id
+ try {
+ const invocationId = `invocation-${randomUUID()}`
+ const reserved = await reserveInDangerZone(
+ {
+ budgetId,
+ limit: 1000,
+ spent: 0,
+ minStart: 0,
+ estAmount: 2000,
+ taskRef: `${invocationId}:1:1`,
+ hostRef: 'research-host',
+ ttlSeconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
+ },
+ pool
+ )
+ if (reserved.decision !== 'allow') throw new Error('expected the reservation to be allowed')
+
+ const attempt = await insertLlmProviderAttempt(pool, {
+ callerKind: 'host',
+ hostRef: 'research-host',
+ invocationId,
+ attemptGeneration: 1,
+ providerAttemptIndex: 1,
+ model: 'gpt-5.1',
+ requestHash: 'b'.repeat(64),
+ policyRevision: 1,
+ policyHash: 'a'.repeat(64),
+ budgetReservationId: reserved.reservationId,
+ connectionRevision: 1,
+ connectionId,
+ })
+ const issued = await issueRegisteredCodexExecutionTicket(pool, {
+ sub: 'host/research-host',
+ hostRef: attempt.hostRef,
+ invocationId: attempt.invocationId,
+ attemptGeneration: attempt.attemptGeneration,
+ providerAttemptId: attempt.id,
+ providerAttemptIndex: attempt.providerAttemptIndex,
+ model: attempt.model,
+ requestHash: attempt.requestHash,
+ policyRevision: attempt.policyRevision,
+ policyHash: attempt.policyHash,
+ budgetReservationId: attempt.budgetReservationId,
+ connectionRevision: attempt.connectionRevision,
+ })
+ await redeemLlmProviderAttempt(
+ { executionTicket: issued.executionTicket, requestHash: attempt.requestHash },
+ txDeps()
+ )
+
+ // Witness: the reservation is live, with the attempt-lifetime TTL, right
+ // before finalize, so its absence afterwards is the release.
+ const before = await pool.query<{ ttl_seconds: number }>(
+ `SELECT EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds
+ FROM budget_pending_reservations
+ WHERE id = $1 AND expires_at > NOW()`,
+ [reserved.reservationId]
+ )
+ expect(before.rows).toEqual([{ ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS }])
+
+ const finalized = await finalizeLlmProviderAttempt(
+ {
+ attemptReceipt: opaqueAttemptReceipt({
+ jti: issued.claims.jti,
+ providerAttemptId: attempt.id,
+ requestHash: attempt.requestHash,
+ }),
+ receipt: {
+ schemaVersion: 'codex-attempt-receipt.v1' as const,
+ providerAttemptId: attempt.id,
+ requestHash: attempt.requestHash,
+ outcome: 'success' as const,
+ usage: { inputTokens: 7, outputTokens: 3 },
+ },
+ },
+ runTx
+ )
+ expect(finalized).toMatchObject({ outcome: 'success', duplicate: false })
+
+ const after = await pool.query<{ count: number }>(
+ `SELECT COUNT(*)::int AS count FROM budget_pending_reservations WHERE id = $1`,
+ [reserved.reservationId]
+ )
+ expect(after.rows[0]?.count).toBe(0)
+ } finally {
+ await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
+ }
+ })
})
diff --git a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts
index a338acfc6..db5f4b37f 100644
--- a/control-api/test/services.llmProviderAttemptStore.helpers.test.ts
+++ b/control-api/test/services.llmProviderAttemptStore.helpers.test.ts
@@ -1,4 +1,8 @@
import { describe, expect, it } from 'vitest'
+import {
+ CODEX_ATTEMPT_MAX_LIFETIME_MS,
+ GROK_ATTEMPT_MAX_LIFETIME_MS,
+} from '../src/services/llmProviderAttemptEnvelope.js'
import {
CODEX_IN_FLIGHT_USAGE_GRACE_MS,
isLinkedCodexInFlightWithoutUsage,
@@ -100,6 +104,25 @@ describe('linked Codex usage predicates', () => {
).toBe(false)
})
+ it('keeps a redeemed row in flight for the whole attempt lifetime of either provider', () => {
+ // A 30 min stream that authorized a minute before its redeem is still
+ // billing; sweeping it would close the invocation under a live stream.
+ for (const lifetime of [CODEX_ATTEMPT_MAX_LIFETIME_MS, GROK_ATTEMPT_MAX_LIFETIME_MS]) {
+ expect(
+ isLinkedCodexInFlightWithoutUsage(
+ {
+ status: 'redeemed',
+ outcome: null,
+ usageInputTokens: null,
+ usageOutputTokens: null,
+ createdAt: new Date(nowMs - lifetime),
+ },
+ nowMs
+ )
+ ).toBe(true)
+ }
+ })
+
it('ages out authorized and redeemed rows after the usage grace', () => {
const stale = new Date(nowMs - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 1)
expect(
diff --git a/control-api/test/services.pluginWorkloadSdkDb.test.ts b/control-api/test/services.pluginWorkloadSdkDb.test.ts
index a82c84454..f2c6ea17a 100644
--- a/control-api/test/services.pluginWorkloadSdkDb.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkDb.test.ts
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { pool } from '../src/db.js'
+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'
import {
PLUGIN_WORKLOAD_SDK_GRANT_UPDATE_CHANNEL,
buildGrantUpdateNotifyPayload,
@@ -583,7 +584,7 @@ describe('invocation retry lifecycle timestamps', () => {
outcome: null,
usage_input_tokens: null,
usage_output_tokens: null,
- created_at: new Date(Date.now() - 16 * 60 * 1000),
+ created_at: new Date(Date.now() - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 60_000),
},
],
rowCount: 1,
diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
index 2babc2f9a..82d293d6b 100644
--- a/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkFinalization.realPostgres.integration.test.ts
@@ -2,7 +2,10 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool, type PoolClient } from 'pg'
import { initDb } from '../src/db.js'
-import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'
+import {
+ CODEX_IN_FLIGHT_USAGE_GRACE_MS,
+ insertLlmProviderAttempt,
+} from '../src/services/llmProviderAttemptStore.js'
import { failStaleInvocationsInTransaction } from '../src/services/pluginWorkloadSdkDb.js'
import {
PromptBridgeFinalizationError,
@@ -597,8 +600,10 @@ describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () =
// Out of the in-flight grace, so the sweeper is free to close it.
const codexAttemptId = await linkCodex(client, input)
await client.query(
- `UPDATE llm_provider_attempts SET created_at = now() - interval '30 minutes' WHERE id = $1`,
- [codexAttemptId]
+ `UPDATE llm_provider_attempts
+ SET created_at = now() - make_interval(secs => $2::double precision / 1000)
+ WHERE id = $1`,
+ [codexAttemptId, CODEX_IN_FLIGHT_USAGE_GRACE_MS + 60_000]
)
expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)
await commit(client)
diff --git a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
index a342bce38..15d104907 100644
--- a/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
+++ b/control-api/test/services.pluginWorkloadSdkFinalization.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from 'vitest'
+import { CODEX_IN_FLIGHT_USAGE_GRACE_MS } from '../src/services/llmProviderAttemptStore.js'
import {
PromptBridgeFinalizationError,
deriveEffectiveSpend,
@@ -1402,7 +1403,10 @@ describe('spend floor writers (Addendum A.4 "best-floor")', () => {
[],
[],
...codexFence('in_progress'),
- linkedCodexRow({ ...codexInFlight, created_at: new Date(Date.now() - 20 * 60_000) }),
+ linkedCodexRow({
+ ...codexInFlight,
+ created_at: new Date(Date.now() - CODEX_IN_FLIGHT_USAGE_GRACE_MS - 60_000),
+ }),
{ rows: [], rowCount: 1 },
{ rows: [], rowCount: 1 },
{ rows: [], rowCount: 1 },