Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(control-api): redeem and finalize Codex attempts
Alfredo Lopez683ab5033af1
diff --git a/control-api/src/app.ts b/control-api/src/app.tsindex fff8ffbde..5d2fdf951 100644--- a/control-api/src/app.ts+++ b/control-api/src/app.ts@@ -22,6 +22,7 @@ import { createInternalAdministrativeEventsRouter } from './routes/internal/admiimport { createInternalAgentRunEventsRouter } from './routes/internal/agentRunEvents.js'import { createInternalBudgetsCheckRouter } from './routes/internal/budgetsCheck.js'import { createInternalInfrastructureTelemetryEventsRouter } from './routes/internal/infrastructureTelemetryEvents.js'+import { createInternalLlmProviderAttemptRoutes } from './routes/internal/llmProviderAttempts.js'import { createInternalOAuthRouter } from './routes/internal/oauth.js'import { createInternalPluginWorkloadSdkRouter } from './routes/internal/pluginWorkloadSdk.js'import { createInternalSandboxUiRouter } from './routes/internal/sandboxUi.js'@@ -241,6 +242,7 @@ export function createApp(gateway: K8sGateway) {// gates the boundary; the public-facing cookie-authed endpoints live in// rpc-proxy and forward here with the user identity asserted in the body.api.use(createInternalOAuthRouter(gateway))+ api.use(createInternalLlmProviderAttemptRoutes())app.use('/api/v1', api)app.use((_req, res) => {diff --git a/control-api/src/config.ts b/control-api/src/config.tsindex 150322452..3e06ee943 100644--- a/control-api/src/config.ts+++ b/control-api/src/config.ts@@ -640,7 +640,7 @@ export const config: Config = {),internalServiceTokens: parseInternalServiceTokens(process.env.CONTROL_API_INTERNAL_SERVICE_TOKENS ||- 'external-rest-api=dev-external-rest-api-token,rpc-proxy=dev-rpc-proxy-token,webhook-proxy=dev-webhook-proxy-token,workflow-approval-reader=dev-wa-reader-token'+ 'external-rest-api=dev-external-rest-api-token,rpc-proxy=dev-rpc-proxy-token,webhook-proxy=dev-webhook-proxy-token,workflow-approval-reader=dev-wa-reader-token,codex-llm-proxy=dev-codex-llm-proxy-token'),internalControlJwtWrcHmacSecret: requiredOrDevDefault('INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET',diff --git a/control-api/src/routes/internal/llmProviderAttempts.ts b/control-api/src/routes/internal/llmProviderAttempts.tsnew file mode 100644index 000000000..b2301314f--- /dev/null+++ b/control-api/src/routes/internal/llmProviderAttempts.ts@@ -0,0 +1,89 @@+import { Router } from 'express'+import { asyncHandler } from '../../http/asyncHandler.js'+import { requireInternalService } from '../../middleware/internalServiceAuth.js'+import { rootLogger } from '../../observability/logger.js'+import {+ LlmProviderAttemptFinalizeError,+ finalizeLlmProviderAttempt,+} from '../../services/llmProviderAttemptFinalization.js'+import {+ LlmProviderAttemptRedeemError,+ redeemLlmProviderAttempt,+} from '../../services/llmProviderAttemptRedemption.js'+import { isPlainObject } from '../../utils/isPlainObject.js'++const log = rootLogger.child({ module: 'internal-llm-provider-attempts' })++const REDEEM_STATUS: Record<string, number> = {+ disabled: 404,+ ticket_invalid: 403,+ ticket_replayed: 409,+ ticket_expired: 403,+ request_hash_mismatch: 403,+ connection_unavailable: 503,+ no_grant: 403,+ provider_unavailable: 503,+}++const FINALIZE_STATUS: Record<string, number> = {+ disabled: 404,+ ticket_invalid: 403,+ request_hash_mismatch: 403,+ invalid_receipt: 400,+ conflict: 409,+}++export function createInternalLlmProviderAttemptRoutes(): Router {+ const router = Router()+ router.use('/internal/llm/provider-attempts', requireInternalService('codex-llm-proxy'))++ router.post(+ '/internal/llm/provider-attempts/redeem',+ asyncHandler(async (req, res) => {+ const body = isPlainObject(req.body) ? req.body : {}+ try {+ const result = await redeemLlmProviderAttempt({+ executionTicket: typeof body.executionTicket === 'string' ? body.executionTicket : '',+ requestHash: typeof body.requestHash === 'string' ? body.requestHash : '',+ model: typeof body.model === 'string' ? body.model : undefined,+ hostRef: typeof body.hostRef === 'string' ? body.hostRef : undefined,+ operation:+ body.operation === 'completion_cancel' || body.operation === 'connection_test'+ ? body.operation+ : 'completion_stream',+ })+ res.status(200).json(result)+ } catch (err) {+ if (err instanceof LlmProviderAttemptRedeemError) {+ log.warn({ event: 'codex_attempt_redeem_denied', code: err.code }, err.message)+ res.status(REDEEM_STATUS[err.code] ?? 400).json({ error: err.code })+ return+ }+ throw err+ }+ })+ )++ router.post(+ '/internal/llm/provider-attempts/finalize',+ asyncHandler(async (req, res) => {+ const body = isPlainObject(req.body) ? req.body : {}+ try {+ const result = await finalizeLlmProviderAttempt({+ attemptReceipt: typeof body.attemptReceipt === 'string' ? body.attemptReceipt : '',+ receipt: body.receipt,+ })+ res.status(200).json(result)+ } catch (err) {+ if (err instanceof LlmProviderAttemptFinalizeError) {+ log.warn({ event: 'codex_attempt_finalize_denied', code: err.code }, err.message)+ res.status(FINALIZE_STATUS[err.code] ?? 400).json({ error: err.code })+ return+ }+ throw err+ }+ })+ )++ return router+}diff --git a/control-api/src/services/llmProviderAttemptFinalization.ts b/control-api/src/services/llmProviderAttemptFinalization.tsnew file mode 100644index 000000000..068fe31b8--- /dev/null+++ b/control-api/src/services/llmProviderAttemptFinalization.ts@@ -0,0 +1,121 @@+import { parseCodexAttemptReceiptV1 } from '@clerum/llm-provider-attempt-contract'+import { config } from '../config.js'+import { withTransaction } from '../db.js'+import { rootLogger } from '../observability/logger.js'+import { hashCodexAttemptReceipt } from './llmProviderAttemptReceipt.js'+import { opaqueAttemptReceipt } from './llmProviderAttemptRedemption.js'+import {+ loadLlmProviderAttempt,+ markLlmProviderAttemptFinalized,+} from './llmProviderAttemptStore.js'++const log = rootLogger.child({ module: 'llm-provider-attempt-finalization' })++export type LlmProviderAttemptFinalizeErrorCode =+ | 'disabled'+ | 'ticket_invalid'+ | 'request_hash_mismatch'+ | 'invalid_receipt'+ | 'conflict'++export class LlmProviderAttemptFinalizeError extends Error {+ constructor(+ readonly code: LlmProviderAttemptFinalizeErrorCode,+ message: string+ ) {+ super(message)+ this.name = 'LlmProviderAttemptFinalizeError'+ }+}++export type FinalizeAttemptInput = {+ attemptReceipt: string+ receipt: unknown+}++export type FinalizeAttemptSuccess = {+ providerAttemptId: string+ outcome: 'success' | 'canceled' | 'error' | 'unknown'+ duplicate: boolean+}++export async function finalizeLlmProviderAttempt(+ input: FinalizeAttemptInput,+ runTransaction: typeof withTransaction = withTransaction+): Promise<FinalizeAttemptSuccess> {+ if (!config.codexSubscriptionEnabled) {+ throw new LlmProviderAttemptFinalizeError('disabled', 'Codex subscription is disabled')+ }+ if (typeof input.attemptReceipt !== 'string' || !/^[a-f0-9]{64}$/.test(input.attemptReceipt)) {+ throw new LlmProviderAttemptFinalizeError('invalid_receipt', 'attemptReceipt is invalid')+ }+ const parsed = parseCodexAttemptReceiptV1(input.receipt)+ if (!parsed.ok) {+ throw new LlmProviderAttemptFinalizeError('invalid_receipt', parsed.message)+ }+ const receipt = parsed.value++ return runTransaction(async tx => {+ const attempt = await loadLlmProviderAttempt(tx, receipt.providerAttemptId)+ if (!attempt) {+ throw new LlmProviderAttemptFinalizeError('ticket_invalid', 'provider attempt was not found')+ }+ if (attempt.requestHash !== receipt.requestHash) {+ throw new LlmProviderAttemptFinalizeError(+ 'request_hash_mismatch',+ 'receipt requestHash does not match the attempt'+ )+ }++ const ticketRow = (+ await tx.query(+ `SELECT jti::text FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1`,+ [attempt.id]+ )+ ).rows[0] as { jti?: string } | undefined+ const jti = ticketRow?.jti ? String(ticketRow.jti) : ''+ const opaque = opaqueAttemptReceipt({+ jti,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ })+ if (opaque !== input.attemptReceipt) {+ throw new LlmProviderAttemptFinalizeError(+ 'ticket_invalid',+ 'attemptReceipt does not authorize this attempt'+ )+ }++ const receiptHash = hashCodexAttemptReceipt(receipt)+ const result = await markLlmProviderAttemptFinalized(tx, {+ providerAttemptId: attempt.id,+ receiptHash,+ outcome: receipt.outcome,+ usageInputTokens: receipt.usage?.inputTokens,+ usageOutputTokens: receipt.usage?.outputTokens,+ })+ if (result === 'missing') {+ throw new LlmProviderAttemptFinalizeError('ticket_invalid', 'provider attempt was not found')+ }+ if (result === 'conflict') {+ throw new LlmProviderAttemptFinalizeError(+ 'conflict',+ 'provider attempt already has a different terminal outcome'+ )+ }+ log.info(+ {+ event: 'codex_attempt_finalized',+ providerAttemptId: attempt.id,+ outcome: receipt.outcome,+ duplicate: result === 'duplicate',+ },+ 'finalized Codex provider attempt'+ )+ return {+ providerAttemptId: attempt.id,+ outcome: receipt.outcome,+ duplicate: result === 'duplicate',+ }+ })+}diff --git a/control-api/src/services/llmProviderAttemptRedemption.ts b/control-api/src/services/llmProviderAttemptRedemption.tsnew file mode 100644index 000000000..53de7b46f--- /dev/null+++ b/control-api/src/services/llmProviderAttemptRedemption.ts@@ -0,0 +1,207 @@+import { createHash } from 'node:crypto'+import { config } from '../config.js'+import { withTransaction } from '../db.js'+import { deriveOAuthEncryptionKey } from '../oauth/encryption.js'+import { rootLogger } from '../observability/logger.js'+import {+ getSafeCodexSubscriptionConnection,+ loadCodexSubscriptionSecrets,+} from './codexSubscriptionConnection.js'+import {+ loadLlmProviderAttempt,+ lockLlmProviderAttemptTicket,+ markLlmProviderAttemptTicketRedeemed,+} from './llmProviderAttemptStore.js'+import { verifyCodexExecutionTicket } from './llmProviderAttemptTicket.js'++const log = rootLogger.child({ module: 'llm-provider-attempt-redemption' })++export const CODEX_COMPLETIONS_ORIGIN = 'https://chatgpt.com/backend-api/codex/responses'+export const CODEX_CATALOG_ORIGIN = 'https://chatgpt.com/backend-api/codex/models'+export const CODEX_TRANSPORT_PROTOCOL = 'codex-subscription-transport.v1'+export const CODEX_MAX_STREAM_DURATION_MS = 300_000++export type LlmProviderAttemptRedeemErrorCode =+ | 'disabled'+ | 'ticket_invalid'+ | 'ticket_replayed'+ | 'ticket_expired'+ | 'request_hash_mismatch'+ | 'connection_unavailable'+ | 'no_grant'+ | 'provider_unavailable'++export class LlmProviderAttemptRedeemError extends Error {+ constructor(+ readonly code: LlmProviderAttemptRedeemErrorCode,+ message: string+ ) {+ super(message)+ this.name = 'LlmProviderAttemptRedeemError'+ }+}++export type RedeemAttemptSuccess = {+ accessToken: string+ transport: {+ protocolVersion: typeof CODEX_TRANSPORT_PROTOCOL+ completionsOrigin: typeof CODEX_COMPLETIONS_ORIGIN+ catalogOrigin: typeof CODEX_CATALOG_ORIGIN+ operation: 'completion_stream' | 'completion_cancel' | 'connection_test'+ servedModel: string+ maxStreamDurationMs: number+ }+ expiryClass: 'short_lived' | 'upstream_managed'+ attemptReceipt: string+}++export type RedeemAttemptInput = {+ executionTicket: string+ requestHash: string+ model?: string+ hostRef?: string+ operation?: 'completion_stream' | 'completion_cancel' | 'connection_test'+}++export function opaqueAttemptReceipt(input: {+ jti: string+ providerAttemptId: string+ requestHash: string+}): string {+ return createHash('sha256')+ .update(`${input.jti}:${input.providerAttemptId}:${input.requestHash}`)+ .digest('hex')+}++export type RedeemAttemptDeps = {+ enabled: boolean+ withTransaction: typeof withTransaction+ loadSecrets: typeof loadCodexSubscriptionSecrets+ encryptionKey: Buffer+}++const defaultRedeemDeps = (): RedeemAttemptDeps => ({+ enabled: config.codexSubscriptionEnabled,+ withTransaction,+ loadSecrets: loadCodexSubscriptionSecrets,+ encryptionKey: deriveOAuthEncryptionKey(config.oauthEncryptionKey),+})++export async function redeemLlmProviderAttempt(+ input: RedeemAttemptInput,+ deps: RedeemAttemptDeps = defaultRedeemDeps()+): Promise<RedeemAttemptSuccess> {+ if (!deps.enabled) {+ throw new LlmProviderAttemptRedeemError('disabled', 'Codex subscription is disabled')+ }+ const claims = verifyCodexExecutionTicket(input.executionTicket)+ if (!claims) {+ throw new LlmProviderAttemptRedeemError('ticket_invalid', 'execution ticket is invalid')+ }+ if (claims.requestHash !== input.requestHash) {+ throw new LlmProviderAttemptRedeemError(+ 'request_hash_mismatch',+ 'requestHash does not match the ticket'+ )+ }+ if (input.model && input.model !== claims.model) {+ throw new LlmProviderAttemptRedeemError('ticket_invalid', 'model does not match the ticket')+ }+ if (input.hostRef && input.hostRef !== claims.hostRef) {+ throw new LlmProviderAttemptRedeemError('ticket_invalid', 'hostRef does not match the ticket')+ }++ return deps.withTransaction(async tx => {+ const ticket = await lockLlmProviderAttemptTicket(tx, claims.jti)+ if (!ticket) {+ throw new LlmProviderAttemptRedeemError(+ 'ticket_invalid',+ 'execution ticket is not registered'+ )+ }+ if (ticket.status !== 'issued') {+ throw new LlmProviderAttemptRedeemError('ticket_replayed', 'execution ticket already used')+ }+ if (ticket.expiresAt.getTime() <= Date.now()) {+ throw new LlmProviderAttemptRedeemError('ticket_expired', 'execution ticket expired')+ }++ const attempt = await loadLlmProviderAttempt(tx, ticket.providerAttemptId)+ if (!attempt || attempt.status !== 'authorized') {+ throw new LlmProviderAttemptRedeemError(+ 'ticket_invalid',+ 'provider attempt is not redeemable'+ )+ }+ if (+ attempt.requestHash !== claims.requestHash ||+ attempt.model !== claims.model ||+ attempt.hostRef !== claims.hostRef ||+ attempt.invocationId !== claims.invocationId ||+ attempt.attemptGeneration !== claims.attemptGeneration ||+ attempt.policyHash !== claims.policyHash ||+ attempt.policyRevision !== claims.policyRevision ||+ attempt.budgetReservationId !== claims.budgetReservationId ||+ attempt.connectionRevision !== claims.connectionRevision+ ) {+ throw new LlmProviderAttemptRedeemError(+ 'no_grant',+ 'ticket bindings no longer match the attempt'+ )+ }++ const connection = await getSafeCodexSubscriptionConnection(tx)+ if (+ !connection ||+ connection.status !== 'connected' ||+ connection.revokedAt ||+ connection.credentialRevision !== attempt.connectionRevision+ ) {+ throw new LlmProviderAttemptRedeemError(+ 'connection_unavailable',+ 'Codex subscription connection is not usable'+ )+ }++ const redeemed = await markLlmProviderAttemptTicketRedeemed(tx, claims.jti)+ if (!redeemed) {+ throw new LlmProviderAttemptRedeemError('ticket_replayed', 'execution ticket already used')+ }++ const secrets = await deps.loadSecrets(tx, deps.encryptionKey)+ if (!secrets?.accessToken) {+ throw new LlmProviderAttemptRedeemError(+ 'connection_unavailable',+ 'no usable access token is available'+ )+ }++ const expiryClass =+ secrets.accessTokenExpiresAt &&+ secrets.accessTokenExpiresAt.getTime() - Date.now() < 3_600_000+ ? 'short_lived'+ : 'upstream_managed'++ log.info(+ { event: 'codex_attempt_redeemed', providerAttemptId: attempt.id },+ 'redeemed Codex execution ticket'+ )+ return {+ accessToken: secrets.accessToken,+ transport: {+ protocolVersion: CODEX_TRANSPORT_PROTOCOL,+ completionsOrigin: CODEX_COMPLETIONS_ORIGIN,+ catalogOrigin: CODEX_CATALOG_ORIGIN,+ operation: input.operation ?? 'completion_stream',+ servedModel: attempt.model,+ maxStreamDurationMs: CODEX_MAX_STREAM_DURATION_MS,+ },+ expiryClass,+ attemptReceipt: opaqueAttemptReceipt({+ jti: claims.jti,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ }),+ }+ })+}diff --git a/control-api/src/services/llmProviderAttemptStore.ts b/control-api/src/services/llmProviderAttemptStore.tsindex 42cd76daa..60dbc0d52 100644--- a/control-api/src/services/llmProviderAttemptStore.ts+++ b/control-api/src/services/llmProviderAttemptStore.ts@@ -167,6 +167,151 @@ export async function registerLlmProviderAttemptTicket()}+export type LlmProviderAttemptTicketRow = {+ jti: string+ providerAttemptId: string+ status: LlmProviderAttemptTicketStatus+ expiresAt: Date+ receiptHash: string | null+}++export async function loadLlmProviderAttempt(+ db: DbClient,+ id: string+): Promise<LlmProviderAttemptRow | null> {+ const result = await db.query(+ `SELECT id, caller_kind, host_ref, recipe_namespace, recipe_name, invocation_id,+ attempt_generation, provider_attempt_index, provider, model, request_hash,+ policy_revision, policy_hash, budget_reservation_id, connection_revision,+ status, outcome, created_at+ FROM llm_provider_attempts+ WHERE id = $1`,+ [id]+ )+ const row = result.rows[0] as Record<string, unknown> | undefined+ if (!row) return null+ return {+ id: String(row.id),+ callerKind: row.caller_kind as 'host' | 'recipe',+ hostRef: String(row.host_ref),+ recipeNamespace: (row.recipe_namespace as string | null) ?? null,+ recipeName: (row.recipe_name as string | null) ?? null,+ invocationId: String(row.invocation_id),+ attemptGeneration: Number(row.attempt_generation),+ providerAttemptIndex: Number(row.provider_attempt_index),+ provider: LLM_PROVIDER_ATTEMPT_PROVIDER,+ model: String(row.model),+ requestHash: String(row.request_hash),+ policyRevision: Number(row.policy_revision),+ policyHash: String(row.policy_hash),+ budgetReservationId: String(row.budget_reservation_id),+ connectionRevision: Number(row.connection_revision),+ status: row.status as LlmProviderAttemptStatus,+ outcome: (row.outcome as LlmProviderAttemptOutcome | null) ?? null,+ createdAt: row.created_at instanceof Date ? row.created_at : new Date(String(row.created_at)),+ }+}++export async function lockLlmProviderAttemptTicket(+ db: DbClient,+ jti: string+): Promise<LlmProviderAttemptTicketRow | null> {+ const result = await db.query(+ `SELECT jti::text, provider_attempt_id::text, status, expires_at, receipt_hash+ FROM llm_provider_attempt_tickets+ WHERE jti = $1+ FOR UPDATE`,+ [jti]+ )+ const row = result.rows[0] as Record<string, unknown> | undefined+ if (!row) return null+ return {+ jti: String(row.jti),+ providerAttemptId: String(row.provider_attempt_id),+ status: row.status as LlmProviderAttemptTicketStatus,+ expiresAt: row.expires_at instanceof Date ? row.expires_at : new Date(String(row.expires_at)),+ receiptHash: (row.receipt_hash as string | null) ?? null,+ }+}++export async function markLlmProviderAttemptTicketRedeemed(+ db: DbClient,+ jti: string+): Promise<boolean> {+ const ticket = await db.query(+ `UPDATE llm_provider_attempt_tickets+ SET status = 'redeemed', redeemed_at = now()+ WHERE jti = $1+ AND status = 'issued'+ AND expires_at > now()`,+ [jti]+ )+ if ((ticket.rowCount ?? 0) !== 1) return false+ await db.query(+ `UPDATE llm_provider_attempts+ SET status = 'redeemed'+ WHERE id = (+ SELECT provider_attempt_id FROM llm_provider_attempt_tickets WHERE jti = $1+ )+ AND status = 'authorized'`,+ [jti]+ )+ return true+}++export async function markLlmProviderAttemptFinalized(+ db: DbClient,+ input: {+ providerAttemptId: string+ receiptHash: string+ outcome: LlmProviderAttemptOutcome+ usageInputTokens?: number+ usageOutputTokens?: number+ }+): Promise<'applied' | 'duplicate' | 'conflict' | 'missing'> {+ const current = await db.query(+ `SELECT status, receipt_hash+ FROM llm_provider_attempt_tickets+ WHERE provider_attempt_id = $1+ FOR UPDATE`,+ [input.providerAttemptId]+ )+ const ticket = current.rows[0] as { status: string; receipt_hash: string | null } | undefined+ if (!ticket) return 'missing'+ if (ticket.status === 'finalized') {+ return ticket.receipt_hash === input.receiptHash ? 'duplicate' : 'conflict'+ }+ if (ticket.status !== 'redeemed') return 'conflict'++ const ticketUpdate = await db.query(+ `UPDATE llm_provider_attempt_tickets+ SET status = 'finalized', finalized_at = now(), receipt_hash = $2+ WHERE provider_attempt_id = $1+ AND status = 'redeemed'`,+ [input.providerAttemptId, input.receiptHash]+ )+ if ((ticketUpdate.rowCount ?? 0) !== 1) return 'conflict'++ const attemptUpdate = await db.query(+ `UPDATE llm_provider_attempts+ SET status = 'finalized',+ outcome = $2,+ usage_input_tokens = $3,+ usage_output_tokens = $4,+ finalized_at = now()+ WHERE id = $1+ AND status = 'redeemed'`,+ [+ input.providerAttemptId,+ input.outcome,+ input.usageInputTokens ?? null,+ input.usageOutputTokens ?? null,+ ]+ )+ if ((attemptUpdate.rowCount ?? 0) !== 1) return 'conflict'+ return 'applied'+}+export async function getMaxLlmProviderAttemptGeneration(db: DbClient,invocationId: stringdiff --git a/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.tsnew file mode 100644index 000000000..da3c9a356--- /dev/null+++ b/control-api/test/services.llmProviderAttemptFinalization.realPostgres.integration.test.ts@@ -0,0 +1,178 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import { config } from '../src/config.js'+import { initDb } from '../src/db.js'+import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'+import {+ insertInitialCodexSubscriptionConnection,+ loadCodexSubscriptionSecrets,+} from '../src/services/codexSubscriptionConnection.js'+import { finalizeLlmProviderAttempt } from '../src/services/llmProviderAttemptFinalization.js'+import { opaqueAttemptReceipt } from '../src/services/llmProviderAttemptRedemption.js'+import { redeemLlmProviderAttempt } from '../src/services/llmProviderAttemptRedemption.js'+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'+import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip+const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++describeRealPostgres('Codex attempt finalization on real PostgreSQL', () => {+ const database = `llm_attempt_final_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ let adminPool: Pool+ let pool: Pool++ async function runTx<T>(work: (tx: Pool) => Promise<T>): Promise<T> {+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const result = await work(client as never)+ await client.query('COMMIT')+ return result+ } catch (error) {+ try {+ await client.query('ROLLBACK')+ } catch {+ // keep the original error+ }+ throw error+ } finally {+ client.release()+ }+ }++ function txDeps() {+ return {+ enabled: true as const,+ encryptionKey: KEY,+ loadSecrets: loadCodexSubscriptionSecrets,+ withTransaction: runTx,+ }+ }++ beforeAll(async () => {+ config.codexSubscriptionEnabled = true+ config.oauthEncryptionKey = 'ab'.repeat(32)+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ pool = new Pool({ connectionString })+ await initDb({ connect: () => pool.connect() })+ await insertInitialCodexSubscriptionConnection(pool, KEY, {+ refreshToken: 'refresh-secret',+ accessToken: 'access-usable',+ accountFingerprint: 'fp-final',+ })+ }, 60_000)++ afterAll(async () => {+ await pool?.end()+ if (adminPool) {+ await adminPool+ .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ .catch(() => undefined)+ await adminPool.end()+ }+ })++ it('accepts repeated finalize calls as one terminal outcome and does not invent usage', async () => {+ const attempt = await insertLlmProviderAttempt(pool, {+ callerKind: 'host',+ hostRef: 'research-host',+ invocationId: `invocation-${randomUUID()}`,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ model: 'gpt-5.1',+ requestHash: 'f'.repeat(64),+ policyRevision: 1,+ policyHash: 'e'.repeat(64),+ budgetReservationId: 'unbudgeted',+ connectionRevision: 1,+ })+ const issued = await issueRegisteredCodexExecutionTicket(pool, {+ sub: 'host/research-host',+ hostRef: attempt.hostRef,+ invocationId: attempt.invocationId,+ attemptGeneration: attempt.attemptGeneration,+ providerAttemptId: attempt.id,+ providerAttemptIndex: attempt.providerAttemptIndex,+ model: attempt.model,+ requestHash: attempt.requestHash,+ policyRevision: attempt.policyRevision,+ policyHash: attempt.policyHash,+ budgetReservationId: attempt.budgetReservationId,+ connectionRevision: attempt.connectionRevision,+ })+ await redeemLlmProviderAttempt(+ {+ executionTicket: issued.executionTicket,+ requestHash: attempt.requestHash,+ },+ txDeps()+ )+ const attemptReceipt = opaqueAttemptReceipt({+ jti: issued.claims.jti,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ })+ const receipt = {+ schemaVersion: 'codex-attempt-receipt.v1' as const,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ outcome: 'unknown' as const,+ }++ const first = await finalizeLlmProviderAttempt({ attemptReceipt, receipt }, runTx)+ const repeats = await Promise.all(+ Array.from({ length: 8 }, () =>+ finalizeLlmProviderAttempt({ attemptReceipt, receipt }, runTx)+ )+ )+ expect(first).toMatchObject({+ providerAttemptId: attempt.id,+ outcome: 'unknown',+ duplicate: false,+ })+ expect(repeats.every(result => result.duplicate && result.outcome === 'unknown')).toBe(true)++ const stored = await pool.query<{+ outcome: string+ usage_input_tokens: number | null+ usage_output_tokens: number | null+ }>(+ `SELECT outcome, usage_input_tokens, usage_output_tokens+ FROM llm_provider_attempts+ WHERE id = $1`,+ [attempt.id]+ )+ expect(stored.rows[0]).toEqual({+ outcome: 'unknown',+ usage_input_tokens: null,+ usage_output_tokens: null,+ })++ await expect(+ finalizeLlmProviderAttempt(+ {+ attemptReceipt,+ receipt: { ...receipt, outcome: 'success' },+ },+ runTx+ )+ ).rejects.toMatchObject({ code: 'conflict' })+ })+})diff --git a/control-api/test/services.llmProviderAttemptFinalization.test.ts b/control-api/test/services.llmProviderAttemptFinalization.test.tsnew file mode 100644index 000000000..5fe8a99b0--- /dev/null+++ b/control-api/test/services.llmProviderAttemptFinalization.test.ts@@ -0,0 +1,32 @@+import { describe, expect, it } from 'vitest'+import { config } from '../src/config.js'+import { finalizeLlmProviderAttempt } from '../src/services/llmProviderAttemptFinalization.js'+import { hashCodexAttemptReceipt } from '../src/services/llmProviderAttemptReceipt.js'++describe('finalizeLlmProviderAttempt', () => {+ it('rejects a malformed receipt without inventing usage', async () => {+ config.codexSubscriptionEnabled = true+ await expect(+ finalizeLlmProviderAttempt({+ attemptReceipt: 'not-a-hash',+ receipt: { outcome: 'success' },+ })+ ).rejects.toMatchObject({ code: 'invalid_receipt' })+ expect(hashCodexAttemptReceipt).toBeTypeOf('function')+ })++ it('is disabled when the flag is off', async () => {+ config.codexSubscriptionEnabled = false+ await expect(+ finalizeLlmProviderAttempt({+ attemptReceipt: 'a'.repeat(64),+ receipt: {+ schemaVersion: 'codex-attempt-receipt.v1',+ providerAttemptId: '33333333-3333-4333-8333-333333333333',+ requestHash: 'b'.repeat(64),+ outcome: 'success',+ },+ })+ ).rejects.toMatchObject({ code: 'disabled' })+ })+})diff --git a/control-api/test/services.llmProviderAttemptRedemption.realPostgres.integration.test.ts b/control-api/test/services.llmProviderAttemptRedemption.realPostgres.integration.test.tsnew file mode 100644index 000000000..0898cb3d9--- /dev/null+++ b/control-api/test/services.llmProviderAttemptRedemption.realPostgres.integration.test.ts@@ -0,0 +1,146 @@+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import { randomBytes, randomUUID } from 'node:crypto'+import { Pool } from 'pg'+import { config } from '../src/config.js'+import { initDb } from '../src/db.js'+import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'+import {+ insertInitialCodexSubscriptionConnection,+ loadCodexSubscriptionSecrets,+} from '../src/services/codexSubscriptionConnection.js'+import { opaqueAttemptReceipt } from '../src/services/llmProviderAttemptRedemption.js'+import { redeemLlmProviderAttempt } from '../src/services/llmProviderAttemptRedemption.js'+import { insertLlmProviderAttempt } from '../src/services/llmProviderAttemptStore.js'+import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'++const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL+const describeRealPostgres = adminUrl ? describe : describe.skip+const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))++function databaseUrl(baseUrl: string, database: string): string {+ const url = new URL(baseUrl)+ url.pathname = `/${database}`+ return url.toString()+}++function quoteIdent(value: string): string {+ return `"${value.replace(/"/g, '""')}"`+}++describeRealPostgres('Codex ticket redemption on real PostgreSQL', () => {+ const database = `llm_attempt_redeem_${randomBytes(6).toString('hex')}`+ const connectionString = databaseUrl(+ adminUrl ?? 'postgresql://[email protected]/postgres',+ database+ )+ let adminPool: Pool+ let pool: Pool++ beforeAll(async () => {+ config.codexSubscriptionEnabled = true+ config.oauthEncryptionKey = 'ab'.repeat(32)+ adminPool = new Pool({ connectionString: adminUrl })+ await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)+ pool = new Pool({ connectionString })+ await initDb({ connect: () => pool.connect() })+ await insertInitialCodexSubscriptionConnection(pool, KEY, {+ refreshToken: 'refresh-secret',+ accessToken: 'access-usable',+ accountFingerprint: 'fp-redeem',+ })+ }, 60_000)++ afterAll(async () => {+ await pool?.end()+ if (adminPool) {+ await adminPool+ .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)+ .catch(() => undefined)+ await adminPool.end()+ }+ })++ it('lets exactly one of 20 concurrent redeems win', async () => {+ const attempt = await insertLlmProviderAttempt(pool, {+ callerKind: 'host',+ hostRef: 'research-host',+ invocationId: `invocation-${randomUUID()}`,+ attemptGeneration: 1,+ providerAttemptIndex: 1,+ model: 'gpt-5.1',+ requestHash: 'd'.repeat(64),+ policyRevision: 1,+ policyHash: 'e'.repeat(64),+ budgetReservationId: 'unbudgeted',+ connectionRevision: 1,+ })+ const issued = await issueRegisteredCodexExecutionTicket(pool, {+ sub: 'host/research-host',+ hostRef: attempt.hostRef,+ invocationId: attempt.invocationId,+ attemptGeneration: attempt.attemptGeneration,+ providerAttemptId: attempt.id,+ providerAttemptIndex: attempt.providerAttemptIndex,+ model: attempt.model,+ requestHash: attempt.requestHash,+ policyRevision: attempt.policyRevision,+ policyHash: attempt.policyHash,+ budgetReservationId: attempt.budgetReservationId,+ connectionRevision: attempt.connectionRevision,+ })++ const results = await Promise.allSettled(+ Array.from({ length: 20 }, () =>+ redeemLlmProviderAttempt(+ {+ executionTicket: issued.executionTicket,+ requestHash: attempt.requestHash,+ },+ {+ enabled: true,+ encryptionKey: KEY,+ loadSecrets: loadCodexSubscriptionSecrets,+ withTransaction: async work => {+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const result = await work(client as never)+ await client.query('COMMIT')+ return result+ } catch (error) {+ try {+ await client.query('ROLLBACK')+ } catch {+ // keep the original error+ }+ throw error+ } finally {+ client.release()+ }+ },+ }+ )+ )+ )+ const wins = results.filter(result => result.status === 'fulfilled')+ const losses = results.filter(result => result.status === 'rejected')+ expect(wins).toHaveLength(1)+ expect(losses).toHaveLength(19)+ if (wins[0]?.status === 'fulfilled') {+ expect(wins[0].value.accessToken).toBe('access-usable')+ expect(JSON.stringify(wins[0].value)).not.toContain('refresh-secret')+ expect(wins[0].value.attemptReceipt).toBe(+ opaqueAttemptReceipt({+ jti: issued.claims.jti,+ providerAttemptId: attempt.id,+ requestHash: attempt.requestHash,+ })+ )+ }+ for (const loss of losses) {+ if (loss.status === 'rejected') {+ expect((loss.reason as { code?: string }).code).toBe('ticket_replayed')+ }+ }+ })+})diff --git a/control-api/test/services.llmProviderAttemptRedemption.test.ts b/control-api/test/services.llmProviderAttemptRedemption.test.tsnew file mode 100644index 000000000..856b8d039--- /dev/null+++ b/control-api/test/services.llmProviderAttemptRedemption.test.ts@@ -0,0 +1,151 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import { config } from '../src/config.js'+import * as connection from '../src/services/codexSubscriptionConnection.js'+import {+ LlmProviderAttemptRedeemError,+ redeemLlmProviderAttempt,+} from '../src/services/llmProviderAttemptRedemption.js'+import * as store from '../src/services/llmProviderAttemptStore.js'+import * as ticket from '../src/services/llmProviderAttemptTicket.js'++vi.mock('../src/db.js', () => ({+ pool: { query: vi.fn() },+ withTransaction: async (work: (tx: { query: typeof vi.fn }) => unknown) =>+ work({ query: vi.fn() }),+}))++const CLAIMS = {+ jti: '11111111-1111-4111-8111-111111111111',+ typ: 'codex-execution-ticket' as const,+ sub: 'host/research-host',+ hostRef: 'research-host',+ invocationId: 'invocation-1',+ attemptGeneration: 1,+ providerAttemptId: '33333333-3333-4333-8333-333333333333',+ providerAttemptIndex: 1,+ provider: 'codex-subscription' as const,+ model: 'gpt-5.1',+ requestHash: 'a'.repeat(64),+ policyRevision: 4,+ policyHash: 'b'.repeat(64),+ budgetReservationId: 'unbudgeted',+ connectionRevision: 3,+}++describe('redeemLlmProviderAttempt', () => {+ beforeEach(() => {+ config.codexSubscriptionEnabled = true+ vi.spyOn(ticket, 'verifyCodexExecutionTicket').mockReturnValue(CLAIMS)+ vi.spyOn(store, 'lockLlmProviderAttemptTicket').mockResolvedValue({+ jti: CLAIMS.jti,+ providerAttemptId: CLAIMS.providerAttemptId,+ status: 'issued',+ expiresAt: new Date(Date.now() + 30_000),+ receiptHash: null,+ })+ vi.spyOn(store, 'loadLlmProviderAttempt').mockResolvedValue({+ id: CLAIMS.providerAttemptId,+ callerKind: 'host',+ hostRef: CLAIMS.hostRef,+ recipeNamespace: null,+ recipeName: null,+ invocationId: CLAIMS.invocationId,+ attemptGeneration: CLAIMS.attemptGeneration,+ providerAttemptIndex: 1,+ provider: 'codex-subscription',+ model: CLAIMS.model,+ requestHash: CLAIMS.requestHash,+ policyRevision: CLAIMS.policyRevision,+ policyHash: CLAIMS.policyHash,+ budgetReservationId: CLAIMS.budgetReservationId,+ connectionRevision: CLAIMS.connectionRevision,+ status: 'authorized',+ outcome: null,+ createdAt: new Date(),+ })+ vi.spyOn(store, 'markLlmProviderAttemptTicketRedeemed').mockResolvedValue(true)+ vi.spyOn(connection, 'getSafeCodexSubscriptionConnection').mockResolvedValue({+ connectionKey: 'deployment-default',+ status: 'connected',+ credentialRevision: 3,+ catalogRevision: 4,+ accountFingerprint: 'fp',+ catalogStatus: 'ready',+ catalogSyncedAt: new Date(),+ lastRefreshAt: null,+ lastAuthAt: new Date(),+ refreshLockHeld: false,+ revokedAt: null,+ createdAt: new Date(),+ updatedAt: new Date(),+ })+ })++ it('returns only the usable access token and frozen transport metadata', async () => {+ const result = await redeemLlmProviderAttempt(+ { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },+ {+ enabled: true,+ withTransaction: async work => work({ query: vi.fn() } as never),+ loadSecrets: async () => ({+ refreshToken: 'refresh-secret',+ accessToken: 'access-usable',+ accessTokenExpiresAt: new Date(Date.now() + 120_000),+ credentialRevision: 3,+ }),+ encryptionKey: Buffer.alloc(32),+ }+ )+ expect(result.accessToken).toBe('access-usable')+ expect(result.transport.completionsOrigin).toBe(+ 'https://chatgpt.com/backend-api/codex/responses'+ )+ expect(JSON.stringify(result)).not.toContain('refresh-secret')+ expect(JSON.stringify(result)).not.toMatch(/encrypted|Authorization/i)+ })++ it('rejects a replayed ticket before returning any token', async () => {+ vi.mocked(store.lockLlmProviderAttemptTicket).mockResolvedValueOnce({+ jti: CLAIMS.jti,+ providerAttemptId: CLAIMS.providerAttemptId,+ status: 'redeemed',+ expiresAt: new Date(Date.now() + 30_000),+ receiptHash: null,+ })+ await expect(+ redeemLlmProviderAttempt(+ { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },+ {+ enabled: true,+ withTransaction: async work => work({ query: vi.fn() } as never),+ loadSecrets: async () => {+ throw new Error('should not load secrets')+ },+ encryptionKey: Buffer.alloc(32),+ }+ )+ ).rejects.toMatchObject({ code: 'ticket_replayed' })+ })++ it('rejects a requestHash mismatch', async () => {+ await expect(+ redeemLlmProviderAttempt({ executionTicket: 'ticket', requestHash: 'c'.repeat(64) })+ ).rejects.toBeInstanceOf(LlmProviderAttemptRedeemError)+ })++ it('is disabled when the flag is off', async () => {+ await expect(+ redeemLlmProviderAttempt(+ { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },+ {+ enabled: false,+ withTransaction: async () => {+ throw new Error('should not run')+ },+ loadSecrets: async () => null,+ encryptionKey: Buffer.alloc(32),+ }+ )+ ).rejects.toMatchObject({ code: 'disabled' })+ })+})