Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { randomBytes, randomUUID } from 'node:crypto'
import { once } from 'node:events'
import { readFileSync } from 'node:fs'
import { createServer as createHttpServer } from 'node:http'
import type { Socket } from 'node:net'
import { setImmediate as nextTurn } from 'node:timers/promises'
import { Pool } from 'pg'
import {
LIMITS as GROK_LIMITS,
computeGrokPolicyHash,
} from '@clerum/grok-provider-attempt-contract'
import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import { config } from '../src/config.js'
import { type DbClient, initDb, withTransaction as withProductionTransaction } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { getActiveReservation } from '../src/services/budgets/reservations.js'
import { getCodexCatalogModelState } from '../src/services/codexSubscriptionCatalog.js'
import {
getSafeCodexSubscriptionConnection,
insertInitialCodexSubscriptionConnection,
recordCodexCatalogOutcome,
} from '../src/services/codexSubscriptionConnection.js'
import type {
IssuedGrokExecutionTicket,
issueRegisteredGrokExecutionTicket,
} from '../src/services/grokProviderAttemptTicket.js'
import {
getSafeGrokSubscriptionConnection,
insertInitialGrokSubscriptionConnection,
recordGrokCatalogOutcome,
} from '../src/services/grokSubscriptionConnection.js'
import {
type LlmProviderAttemptAuthorizerDeps,
authorizeLlmProviderAttempt,
computeCodexPolicyHash,
} from '../src/services/llmProviderAttemptAuthorizer.js'
import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'
import {
getMaxLlmProviderAttemptGeneration,
insertLlmProviderAttempt,
} from '../src/services/llmProviderAttemptStore.js'
import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'
import { ResourceService } from '../src/services/resourceService.js'
import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'
// The Grok authorizer signs through this module, not through a dependency, so
// the Grok envelope test taps the real issuer here: the ticket is registered
// for real, then `afterIssue` may replace the returned value.
type IssueGrokTicket = typeof issueRegisteredGrokExecutionTicket
const grokTicketTap = vi.hoisted(() => ({
afterIssue: null as
| null
| ((
db: Parameters<IssueGrokTicket>[0],
input: Parameters<IssueGrokTicket>[1],
issued: IssuedGrokExecutionTicket
) => Promise<IssuedGrokExecutionTicket>),
}))
vi.mock('../src/services/grokProviderAttemptTicket.js', async importOriginal => {
const actual =
await importOriginal<typeof import('../src/services/grokProviderAttemptTicket.js')>()
return {
...actual,
issueRegisteredGrokExecutionTicket: async (...args: Parameters<IssueGrokTicket>) => {
const issued = await actual.issueRegisteredGrokExecutionTicket(...args)
return grokTicketTap.afterIssue ? grokTicketTap.afterIssue(args[0], args[1], issued) : issued
},
}
})
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))
function databaseUrl(baseUrl: string, database: string): string {
const url = new URL(baseUrl)
url.pathname = `/${database}`
return url.toString()
}
function quoteIdent(value: string): string {
return `"${value.replace(/"/g, '""')}"`
}
async function cancellationDeadline<T>(promise: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout | undefined
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(
() => reject(new Error('authorizer cancellation phase did not complete')),
2_000
)
}),
])
} finally {
clearTimeout(timer)
}
}
async function observeCancellation(check: () => Promise<boolean>): Promise<void> {
const deadline = Date.now() + 2_000
while (!(await check())) {
if (Date.now() >= deadline) throw new Error('authorizer physical state did not settle')
await nextTurn()
}
}
async function createAssignmentLookupPeer(stall: boolean) {
let arrived!: () => void
const arrival = new Promise<void>(resolve => {
arrived = resolve
})
const sockets = new Set<Socket>()
const server = createHttpServer((req, res) => {
arrived()
if (stall) return
res.setHeader('Content-Type', 'application/json')
res.end(JSON.stringify({ metadata: { name: 'research-host' } }))
})
server.on('connection', socket => {
sockets.add(socket)
socket.once('close', () => sockets.delete(socket))
})
server.listen(0, '127.0.0.1')
await once(server, 'listening')
const address = server.address()
if (!address || typeof address === 'string') throw new Error('missing assignment peer address')
const kubeconfig = new k8s.KubeConfig()
kubeconfig.loadFromOptions({
clusters: [
// client-node requires an explicit opt-in for this owned loopback HTTP peer.
{ name: 'fixture', server: `http://127.0.0.1:${address.port}`, skipTLSVerify: true },
],
users: [{ name: 'fixture' }],
contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],
currentContext: 'fixture',
})
return {
resources: new ResourceService(kubeconfig.makeApiClient(k8s.CustomObjectsApi), 'fixture', {
hosts: 'fixture',
}),
arrived: arrival,
get openSocketCount() {
return sockets.size
},
recover: () => {
stall = false
},
close: async () => {
for (const socket of sockets) socket.destroy()
await new Promise<void>((resolve, reject) =>
server.close(error => (error ? reject(error) : resolve()))
)
},
}
}
const REQUEST = {
schemaVersion: 'codex-completion-request.v1' as const,
requestId: 'req-pg-1',
idempotencyKey: 'idem-pg-1',
provider: 'codex-subscription' as const,
model: 'gpt-5.1',
messages: [{ role: 'user' as const, content: 'hello from postgres' }],
}
function claims(): McpHostAccessClaims {
return {
sub: 'default/research-host',
recipeNamespace: 'default',
recipeName: 'research-host',
hostRefs: ['research-host'],
scope: 'workflow:approval:request',
workflowControlScopes: ['llm:codex:execute'],
mcpCapabilities: [],
iss: 'control-api',
aud: 'workflow-approvals',
jti: randomUUID(),
exp: Math.floor(Date.now() / 1000) + 60,
}
}
describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL', () => {
const database = `llm_attempt_authz_${randomBytes(6).toString('hex')}`
const connectionString = databaseUrl(
adminUrl ?? `postgresql://postgres@${['127', '0', '0', '1'].join('.')}/postgres`,
database
)
let adminPool: Pool
let pool: Pool
function dbClient() {
return { query: (text: string, values?: unknown[]) => pool.query(text, values) }
}
function testDeps(
overrides: Partial<LlmProviderAttemptAuthorizerDeps> = {}
): LlmProviderAttemptAuthorizerDeps {
const db = dbClient()
return {
enabled: true,
db,
withTransaction: async work => {
const client = await pool.connect()
try {
await client.query('BEGIN')
const result = await work(client as never)
await client.query('COMMIT')
return result
} catch (error) {
try {
await client.query('ROLLBACK')
} catch {
// keep the original error
}
throw error
} finally {
client.release()
}
},
getConnection: getSafeCodexSubscriptionConnection,
getModelState: getCodexCatalogModelState,
resolveAssignment: async () => ({
liveBrokerProviders: ['codex-subscription'],
liveConnectionRef: 'deployment-default',
}),
evaluateBudget: evaluateBudgetCheck,
getActiveReservation,
getMaxGeneration: getMaxLlmProviderAttemptGeneration,
insertAttempt: insertLlmProviderAttempt,
issueTicket: issueRegisteredCodexExecutionTicket,
...overrides,
}
}
function registerProductionCancellationCases(fixture: {
provider: 'codex-subscription' | 'grok-subscription'
model: string
connectionKey: string
caller: () => McpHostAccessClaims
request: () => Record<string, unknown>
readConnection: () => Promise<{
id: string
catalogRevision: number
credentialRevision: number
connectionKey: string
} | null>
policyHash: (connection: {
catalogRevision: number
credentialRevision: number
connectionKey: string
}) => string
}): void {
describe(`${fixture.provider} production cancellation`, () => {
async function run(stage: 'assignment' | 'query' | 'commit'): Promise<void> {
const invocationId = `actual-cancel-${randomUUID()}`
const budgetId = randomUUID()
const current = await fixture.readConnection()
expect(current).not.toBeNull()
const payload = {
request: fixture.request(),
invocationId,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: fixture.policyHash(current!),
}
const observer = new Pool({
connectionString,
max: 1,
connectionTimeoutMillis: 2_000,
statement_timeout: 2_000,
idleTimeoutMillis: 1_000,
})
const proxy =
stage === 'commit'
? await createPostgresCommitReplyBlackhole(connectionString)
: undefined
const workPool = new Pool({
connectionString: proxy?.connectionString ?? connectionString,
ssl: false,
max: 1,
connectionTimeoutMillis: 2_000,
statement_timeout: 15_000,
idleTimeoutMillis: 1_000,
})
const lookup = await createAssignmentLookupPeer(stage === 'assignment')
const controller = new AbortController()
const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
const phases: string[] = []
let backendPid: number | undefined
let attemptId: string | undefined
let reservationId: string | undefined
let stallQuery = stage === 'query'
let settled = false
const previousTap = grokTicketTap.afterIssue
async function counts() {
return (
await observer.query<{ attempts: number; tickets: number; reservations: number }>(
`SELECT
(SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,
(SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a
ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,
(SELECT COUNT(*)::int FROM budget_pending_reservations WHERE budget_id = $2) AS reservations`,
[invocationId, budgetId]
)
).rows[0]
}
async function afterRealIssue(
db: DbClient,
input: { providerAttemptId: string; budgetReservationId: string },
issued: { claims: { jti: string } }
): Promise<void> {
const registered = await db.query(
`SELECT a.provider, a.connection_id, a.budget_reservation_id,
t.jti, t.status AS ticket_status, r.budget_id, r.host_ref
FROM llm_provider_attempts a
JOIN llm_provider_attempt_tickets t ON t.provider_attempt_id = a.id
JOIN budget_pending_reservations r ON r.id::text = a.budget_reservation_id
WHERE a.id = $1`,
[input.providerAttemptId]
)
expect(registered.rows).toEqual([
{
provider: fixture.provider,
connection_id: current!.id,
budget_reservation_id: input.budgetReservationId,
jti: issued.claims.jti,
ticket_status: 'issued',
budget_id: budgetId,
host_ref: fixture.caller().hostRefs[0],
},
])
attemptId = input.providerAttemptId
reservationId = input.budgetReservationId
phases.push('ticket_registered')
// The real issuer has already signed and registered its ticket. This
// native statement supplies a deterministic interruptible phase; no
// test barrier resolves the work after cancellation.
if (stallQuery) await db.query('SELECT pg_sleep(30)')
}
function productionDeps(signal: AbortSignal): LlmProviderAttemptAuthorizerDeps {
if (fixture.provider === 'grok-subscription') {
grokTicketTap.afterIssue = async (db, input, issued) => {
await afterRealIssue(db, input, issued)
return issued
}
}
return testDeps({
signal,
resolveAssignment: async (_hostRef, forwarded) => {
expect(forwarded).toBe(signal)
phases.push('assignment')
await lookup.resources.getResource('hosts', 'research-host', 'fixture', forwarded)
return {
liveBrokerProviders: [fixture.provider],
liveConnectionRef: fixture.connectionKey,
}
},
withTransaction: async (work, _unused, options) => {
// Assert the authorizer's actual forwarding before binding this
// fixture database; do not inject a separate cancellation source.
expect(options?.signal).toBe(signal)
return withProductionTransaction(
async tx => {
phases.push('transaction')
const identity = await tx.query('SELECT pg_backend_pid() AS pid')
backendPid = (identity.rows[0] as { pid: number }).pid
return work(tx)
},
workPool,
options
)
},
evaluateBudget: async (...args) => {
const result = await evaluateBudgetCheck(...args)
expect(result.allowed).toBe(true)
expect(result.reservationIds).toHaveLength(1)
phases.push('budget_reserved')
return result
},
insertAttempt: async (db, input) => {
const attempt = await insertLlmProviderAttempt(db, input)
phases.push('attempt_inserted')
return attempt
},
issueTicket: async (db, input) => {
const issued = await issueRegisteredCodexExecutionTicket(db, input)
await afterRealIssue(db, input, issued)
return issued
},
})
}
try {
const runtime = (
await observer.query<{ version: string; version_num: string }>(
"SELECT version(), current_setting('server_version_num') AS version_num"
)
).rows[0]
expect(Number(runtime.version_num)).toBeGreaterThanOrEqual(160000)
expect(Number(runtime.version_num)).toBeLessThan(170000)
expect(runtime.version).toMatch(/linux|darwin|bsd|illumos/i)
await observer.query(
`INSERT INTO token_budgets
(id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
VALUES ($1, 'actual authorize cancellation', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
[budgetId, JSON.stringify({ provider: [fixture.provider], model: [fixture.model] })]
)
__resetBudgetCheckCache()
const outcome = authorizeLlmProviderAttempt(
fixture.caller(),
payload,
productionDeps(controller.signal)
)
.then(
value => ({ value }),
error => ({ error })
)
.finally(() => {
settled = true
})
async function beforeCancellation<T>(phase: Promise<T>): Promise<T> {
// Report an actual authorization failure instead of timing out on a
// phase that the rejected production path can no longer reach.
const unexpectedCompletion = outcome.then(result => {
if ('error' in result) throw result.error
throw new Error('authorizer completed before its cancellation phase')
})
if (settled) return unexpectedCompletion
return cancellationDeadline(Promise.race([phase, unexpectedCompletion]))
}
const expectedPhases = [
'assignment',
'transaction',
'budget_reserved',
'attempt_inserted',
'ticket_registered',
]
if (stage === 'assignment') {
await beforeCancellation(lookup.arrived)
expect(phases).toEqual(['assignment'])
expect(workPool.totalCount).toBe(0)
expect(lookup.openSocketCount).toBe(1)
} else if (stage === 'query') {
await observeCancellation(async () => {
if (!backendPid) return beforeCancellation(Promise.resolve(false))
const result = await beforeCancellation(
observer.query<{ state: string; query: string }>(
'SELECT state, query FROM pg_stat_activity WHERE pid = $1',
[backendPid]
)
)
return (
result.rows[0]?.state === 'active' &&
result.rows[0]?.query === 'SELECT pg_sleep(30)'
)
})
expect(phases).toEqual(expectedPhases)
expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
} else {
await beforeCancellation(proxy!.commitForwarded)
await observeCancellation(
async () => (await beforeCancellation(counts())).attempts === 1
)
await observeCancellation(async () => proxy!.discardedReplyBytes > 0)
expect(phases).toEqual(expectedPhases)
expect(proxy!.commands).toEqual(['BEGIN', 'COMMIT'])
expect(proxy!.discardedReplyBytes).toBeGreaterThan(0)
}
expect(settled).toBe(false)
controller.abort(reason)
const interrupted = await cancellationDeadline(outcome)
expect('error' in interrupted ? interrupted.error : undefined).toBe(reason)
expect(workPool.totalCount).toBe(0)
if (stage === 'assignment') {
await observeCancellation(async () => lookup.openSocketCount === 0)
}
if (backendPid) {
await observeCancellation(
async () =>
(
await observer.query<{ count: number }>(
'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1',
[backendPid]
)
).rows[0].count === 0
)
}
await withProductionTransaction(async db => {
const result = await db.query(
'SELECT pg_try_advisory_xact_lock(hashtext($1)) AS available',
[budgetId]
)
expect((result.rows[0] as { available: boolean }).available).toBe(true)
}, observer)
if (stage === 'commit') {
// The client never received COMMIT acknowledgement, yet the real
// business ledger is durable. Do not label this rollback/no spend.
expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
const attempt = await observer.query(
'SELECT id, provider, connection_id, budget_reservation_id, status FROM llm_provider_attempts WHERE invocation_id = $1',
[invocationId]
)
expect(attempt.rows).toEqual([
{
id: attemptId,
provider: fixture.provider,
connection_id: current!.id,
budget_reservation_id: reservationId,
status: 'authorized',
},
])
} else {
expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
stallQuery = false
lookup.recover()
phases.length = 0
const later = await authorizeLlmProviderAttempt(
fixture.caller(),
payload,
productionDeps(new AbortController().signal)
)
expect(later.providerAttemptId).toBe(attemptId)
expect(phases).toEqual(expectedPhases)
expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
expect(workPool.idleCount).toBe(1)
}
} finally {
controller.abort(reason)
grokTicketTap.afterIssue = previousTap
proxy?.allowReplies()
await endPoolAndWaitForClients(workPool)
await proxy?.close()
await lookup.close()
await observer.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
__resetBudgetCheckCache()
await endPoolAndWaitForClients(observer)
}
}
it(
'cancels a configured Kubernetes lookup before any production transaction, then reauthorizes the same binding',
() => run('assignment'),
10_000
)
it(
'cancels a native query after real budget, attempt and signed ticket writes, then rolls back and reauthorizes',
() => run('query'),
10_000
)
it(
'keeps real authorization ledger rows when COMMIT executes but its reply is lost',
() => run('commit'),
10_000
)
})
}
beforeAll(async () => {
adminPool = new Pool({ connectionString: adminUrl })
await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
pool = new Pool({ connectionString })
await initDb({ connect: () => pool.connect() })
const created = await insertInitialCodexSubscriptionConnection(pool, KEY, {
refreshToken: 'refresh-authz',
accountFingerprint: 'fp-authz',
})
await recordCodexCatalogOutcome(pool, {
catalogStatus: 'ready',
expectedCredentialRevision: 1,
expectedCatalogRevision: 0,
})
await pool.query(
`INSERT INTO codex_catalog_models
(connection_id, model, enabled, source, discovered_at, last_seen_at, stale)
VALUES ($1, 'gpt-5.1', true, 'discovery', NOW(), NOW(), false)`,
[created.id]
)
}, 60_000)
afterAll(async () => {
try {
await pool?.end()
if (adminPool) {
await adminPool
.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
.catch(() => undefined)
}
} finally {
await adminPool?.end()
}
})
it('lets only one concurrent authorization win for the same invocation binding', async () => {
const current = await getSafeCodexSubscriptionConnection(pool)
expect(current?.status).toBe('connected')
const payload = {
request: REQUEST,
invocationId: `invocation-${randomUUID()}`,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: computeCodexPolicyHash({
model: REQUEST.model,
catalogRevision: current!.catalogRevision,
credentialRevision: current!.credentialRevision,
connectionKey: current!.connectionKey,
}),
}
const deps = testDeps()
const [first, second] = await Promise.allSettled([
authorizeLlmProviderAttempt(claims(), payload, deps),
authorizeLlmProviderAttempt(claims(), payload, deps),
])
const wins = [first, second].filter(result => result.status === 'fulfilled')
const losses = [first, second].filter(result => result.status === 'rejected')
expect(wins).toHaveLength(1)
expect(losses).toHaveLength(1)
if (losses[0]?.status === 'rejected') {
expect((losses[0].reason as { code?: string }).code).toBe('idempotency_conflict')
}
const count = await pool.query<{ count: string }>(
`SELECT COUNT(*)::text AS count
FROM llm_provider_attempts
WHERE invocation_id = $1`,
[payload.invocationId]
)
expect(count.rows[0]?.count).toBe('1')
expect(JSON.stringify(wins[0])).not.toContain('hello from postgres')
})
it('rolls budget reservation, attempt, and ticket back together when ticket issue fails', async () => {
const current = await getSafeCodexSubscriptionConnection(pool)
const invocationId = `invocation-rollback-${randomUUID()}`
const payload = {
request: { ...REQUEST, requestId: 'req-pg-2', idempotencyKey: 'idem-pg-2' },
invocationId,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: computeCodexPolicyHash({
model: REQUEST.model,
catalogRevision: current!.catalogRevision,
credentialRevision: current!.credentialRevision,
connectionKey: current!.connectionKey,
}),
}
await expect(
authorizeLlmProviderAttempt(
claims(),
payload,
testDeps({
issueTicket: async () => {
throw new Error('ticket registration failed')
},
})
)
).rejects.toThrow(/ticket registration failed/)
const leftover = await pool.query<{ count: string }>(
`SELECT COUNT(*)::text AS count
FROM llm_provider_attempts
WHERE invocation_id = $1`,
[invocationId]
)
expect(leftover.rows[0]?.count).toBe('0')
})
it('reserves a danger-zone token budget for the whole attempt lifetime', async () => {
const current = await getSafeCodexSubscriptionConnection(pool)
// remaining (1000) < max_task_amount (2000) puts the budget in the danger
// zone, and min_start 0 lets the attempt through with a reservation.
const budget = await pool.query<{ id: string }>(
`INSERT INTO token_budgets
(name, scope, unit, limit_amount, period, timezone,
min_start_amount, max_task_amount, enforcement)
VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')
RETURNING id`
)
const budgetId = budget.rows[0]!.id
__resetBudgetCheckCache()
try {
const invocationId = `invocation-envelope-${randomUUID()}`
const authorized = await authorizeLlmProviderAttempt(
claims(),
{
request: { ...REQUEST, requestId: 'req-pg-3', idempotencyKey: 'idem-pg-3' },
invocationId,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: computeCodexPolicyHash({
model: REQUEST.model,
catalogRevision: current!.catalogRevision,
credentialRevision: current!.credentialRevision,
connectionKey: current!.connectionKey,
}),
},
testDeps()
)
expect(authorized.providerAttemptId).toEqual(expect.any(String))
// created_at and expires_at come from the same statement's NOW(), so
// their difference is exactly the TTL the reservation was written with.
const reservations = await pool.query<{
id: string
task_ref: string
ttl_seconds: number
}>(
`SELECT id, task_ref,
EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds
FROM budget_pending_reservations
WHERE budget_id = $1`,
[budgetId]
)
expect(reservations.rows).toEqual([
{
id: expect.any(String),
task_ref: `${invocationId}:1:1`,
ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
},
])
expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)
const attempt = await pool.query<{ budget_reservation_id: string }>(
`SELECT budget_reservation_id FROM llm_provider_attempts WHERE invocation_id = $1`,
[invocationId]
)
expect(attempt.rows).toEqual([{ budget_reservation_id: reservations.rows[0]!.id }])
} finally {
await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
__resetBudgetCheckCache()
}
})
it.each([false, true])(
'rolls back an oversized signed V2 envelope without losing an existing reservation (presented=%s)',
async presented => {
const current = await getSafeCodexSubscriptionConnection(pool)
const invocationId = `visual-envelope-${randomUUID()}`
const request = {
...REQUEST,
schemaVersion: 'codex-completion-request.v2',
messages: [{ role: 'user', content: '' }],
}
const budgetId = randomUUID()
const existingReservationId = randomUUID()
// Force the real danger-zone path without relying on rollup timing. The
// configured maximum task is greater than the remaining period allowance.
await pool.query(
`INSERT INTO token_budgets
(id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
VALUES ($1, 'visual envelope rollback', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
[budgetId, JSON.stringify({ provider: ['codex-subscription'], model: [REQUEST.model] })]
)
await pool.query(
`INSERT INTO budget_pending_reservations
(id, budget_id, est_amount, task_ref, host_ref, expires_at)
VALUES ($1, $2, 5, 'preexisting-visual-task', 'research-host', NOW() + INTERVAL '15 minutes')`,
[existingReservationId, budgetId]
)
__resetBudgetCheckCache()
const payload = {
request,
invocationId,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: computeCodexPolicyHash({
model: REQUEST.model,
catalogRevision: current!.catalogRevision,
credentialRevision: current!.credentialRevision,
connectionKey: current!.connectionKey,
}),
...(presented ? { budgetReservationId: existingReservationId } : {}),
}
request.messages[0].content = 'x'.repeat(
LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(payload)) - 16
)
const counts = () =>
pool.query(`SELECT
(SELECT count(*)::text FROM llm_provider_attempts) AS attempts,
(SELECT count(*)::text FROM llm_provider_attempt_tickets) AS tickets,
(SELECT count(*)::text FROM budget_pending_reservations) AS reservations`)
const readExisting = () =>
pool.query('SELECT * FROM budget_pending_reservations WHERE id = $1', [
existingReservationId,
])
const before = (await counts()).rows[0]
const existingBefore = (await readExisting()).rows[0]
let observedReservationId: string | undefined
let signed = false
try {
await expect(
authorizeLlmProviderAttempt(
claims(),
payload,
testDeps({
issueTicket: async (db, input) => {
observedReservationId = input.budgetReservationId
const active = await db.query(
'SELECT id FROM budget_pending_reservations WHERE budget_id = $1',
[budgetId]
)
expect(active.rows).toHaveLength(presented ? 1 : 2)
if (presented) expect(input.budgetReservationId).toBe(existingReservationId)
else {
expect(input.budgetReservationId).not.toBe('unbudgeted')
expect(input.budgetReservationId).not.toBe(existingReservationId)
}
const issued = await issueRegisteredCodexExecutionTicket(db, input)
const registered = await db.query(
'SELECT jti FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1',
[input.providerAttemptId]
)
expect(registered.rows).toHaveLength(1)
signed = true
// The request's text budget stays at 1 MiB. Inject an oversized
// return value only after a real ticket/reservation exists in
// this transaction, to prove the final envelope guard rolls
// back database writes. This value never leaves the test.
return {
...issued,
executionTicket:
issued.executionTicket + 'x'.repeat(LIMITS.maxVisualRequestBodyBytes),
}
},
})
)
).rejects.toMatchObject({ code: 'payload_too_large' })
expect(signed).toBe(true)
expect((await counts()).rows[0]).toEqual(before)
expect((await readExisting()).rows[0]).toEqual(existingBefore)
if (!presented) {
expect(observedReservationId).toBeDefined()
expect(
(
await pool.query('SELECT id FROM budget_pending_reservations WHERE id = $1', [
observedReservationId,
])
).rows
).toHaveLength(0)
}
const leftover = await pool.query(
'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
[invocationId]
)
expect(leftover.rows).toHaveLength(0)
// A failed size check must not burn the invocation/attempt identity.
// Retrying that same binding with a request that fits must authorize.
request.messages[0].content = 'A bounded request after rollback'
const retried = await authorizeLlmProviderAttempt(claims(), payload, testDeps())
const committed = await pool.query(
'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
[invocationId]
)
expect(committed.rows).toEqual([{ id: retried.providerAttemptId }])
expect((await readExisting()).rows[0]).toEqual(existingBefore)
} finally {
await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
__resetBudgetCheckCache()
}
}
)
registerProductionCancellationCases({
provider: 'codex-subscription',
model: REQUEST.model,
connectionKey: 'deployment-default',
caller: claims,
readConnection: () => getSafeCodexSubscriptionConnection(pool),
request: () => ({
...REQUEST,
schemaVersion: 'codex-completion-request.v2',
requestId: randomUUID(),
idempotencyKey: randomUUID(),
}),
policyHash: connection =>
computeCodexPolicyHash({
model: REQUEST.model,
catalogRevision: connection.catalogRevision,
credentialRevision: connection.credentialRevision,
connectionKey: connection.connectionKey,
}),
})
// #784: the Grok twin of the Codex rollback above, on an image-bearing V2
// request. The Grok path checks its proxy envelope after signing and before
// commit, so a refusal there must roll back the attempt, the registered
// ticket and any new reservation.
describe('Grok V2 proxy envelope', () => {
const GROK_CONNECTION_KEY = 'team-grok-visual'
const GROK_MODEL = 'grok-4.6'
const previousGrokFlag = config.grokSubscriptionEnabled
function grokClaims(): McpHostAccessClaims {
return { ...claims(), workflowControlScopes: ['llm:grok:execute'] }
}
/** The real 2x2 PNG part of the Grok contract's V2 fixture. */
function fixturePngPart(): unknown {
const fixture = JSON.parse(
readFileSync(
new URL(
'../../packages/grok-provider-attempt-contract/fixtures/visual-requests.json',
import.meta.url
),
'utf8'
)
) as { png: { messages: Array<{ contentParts: Array<{ type: string }> }> } }
const part = fixture.png.messages[0]!.contentParts.find(item => item.type === 'image')
expect(part).toBeDefined()
return part
}
beforeAll(async () => {
const created = await insertInitialGrokSubscriptionConnection(
pool,
KEY,
{ refreshToken: 'refresh-grok-visual', accountFingerprint: 'fp-grok-visual' },
GROK_CONNECTION_KEY
)
await recordGrokCatalogOutcome(pool, {
catalogStatus: 'ready',
connectionStatus: 'connected',
expectedCredentialRevision: 1,
expectedCatalogRevision: 0,
connectionKey: GROK_CONNECTION_KEY,
})
await pool.query(
`INSERT INTO grok_catalog_models
(connection_id, model, enabled, source, discovered_at, last_seen_at, stale)
VALUES ($1, $2, true, 'discovery', NOW(), NOW(), false)`,
[created.id, GROK_MODEL]
)
})
// Per-test flag and tap state: the file's only afterAll owns the
// administrative pool close (realPostgresAdminCleanup.guard.test.ts).
beforeEach(() => {
config.grokSubscriptionEnabled = true
})
afterEach(() => {
config.grokSubscriptionEnabled = previousGrokFlag
grokTicketTap.afterIssue = null
})
registerProductionCancellationCases({
provider: 'grok-subscription',
model: GROK_MODEL,
connectionKey: GROK_CONNECTION_KEY,
caller: grokClaims,
readConnection: () => getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY),
request: () => ({
schemaVersion: 'grok-completion-request.v2',
requestId: randomUUID(),
idempotencyKey: randomUUID(),
provider: 'grok-subscription',
model: GROK_MODEL,
messages: [
{
role: 'user',
content: 'look',
contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],
},
],
}),
policyHash: connection =>
computeGrokPolicyHash({
model: GROK_MODEL,
catalogRevision: connection.catalogRevision,
credentialRevision: connection.credentialRevision,
connectionKey: GROK_CONNECTION_KEY,
}),
})
it.each([false, true])(
'rolls back an oversized signed Grok V2 envelope without losing an existing reservation (presented=%s)',
async presented => {
const current = await getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY)
expect(current?.status).toBe('connected')
const invocationId = `grok-visual-envelope-${randomUUID()}`
const request = {
schemaVersion: 'grok-completion-request.v2',
requestId: `req-${invocationId}`,
idempotencyKey: `idem-${invocationId}`,
provider: 'grok-subscription',
model: GROK_MODEL,
messages: [
{
role: 'user',
content: 'look',
contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],
},
],
}
const budgetId = randomUUID()
const existingReservationId = randomUUID()
await pool.query(
`INSERT INTO token_budgets
(id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
VALUES ($1, 'grok visual envelope rollback', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
[budgetId, JSON.stringify({ provider: ['grok-subscription'], model: [GROK_MODEL] })]
)
await pool.query(
`INSERT INTO budget_pending_reservations
(id, budget_id, est_amount, task_ref, host_ref, expires_at)
VALUES ($1, $2, 5, 'preexisting-grok-visual-task', 'research-host', NOW() + INTERVAL '15 minutes')`,
[existingReservationId, budgetId]
)
__resetBudgetCheckCache()
const payload = {
request,
invocationId,
attemptGeneration: 1,
providerAttemptIndex: 1,
policyRevision: current!.catalogRevision,
policyHash: computeGrokPolicyHash({
model: GROK_MODEL,
catalogRevision: current!.catalogRevision,
credentialRevision: current!.credentialRevision,
connectionKey: GROK_CONNECTION_KEY,
}),
...(presented ? { budgetReservationId: existingReservationId } : {}),
}
const grokDeps = () =>
testDeps({
resolveAssignment: async () => ({
liveBrokerProviders: ['grok-subscription'],
liveConnectionRef: GROK_CONNECTION_KEY,
}),
})
const counts = () =>
pool.query(`SELECT
(SELECT count(*)::text FROM llm_provider_attempts) AS attempts,
(SELECT count(*)::text FROM llm_provider_attempt_tickets) AS tickets,
(SELECT count(*)::text FROM budget_pending_reservations) AS reservations`)
const readExisting = () =>
pool.query('SELECT * FROM budget_pending_reservations WHERE id = $1', [
existingReservationId,
])
const before = (await counts()).rows[0]
const existingBefore = (await readExisting()).rows[0]
let observedReservationId: string | undefined
let signed = false
grokTicketTap.afterIssue = async (db, input, issued) => {
observedReservationId = input.budgetReservationId
const active = await db.query(
'SELECT id FROM budget_pending_reservations WHERE budget_id = $1',
[budgetId]
)
expect(active.rows).toHaveLength(presented ? 1 : 2)
if (presented) expect(input.budgetReservationId).toBe(existingReservationId)
else {
expect(input.budgetReservationId).not.toBe('unbudgeted')
expect(input.budgetReservationId).not.toBe(existingReservationId)
}
const registered = await db.query(
'SELECT jti FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1',
[input.providerAttemptId]
)
expect(registered.rows).toEqual([{ jti: issued.claims.jti }])
signed = true
// No real image-bearing request can exceed the 35 MiB envelope once
// its images and text are inside their own budgets. Inflate the
// returned ticket only after the real ticket and reservation exist
// in this transaction. This value never leaves the test.
return {
...issued,
executionTicket:
issued.executionTicket + 'x'.repeat(GROK_LIMITS.maxVisualRequestBodyBytes),
}
}
try {
await expect(
authorizeLlmProviderAttempt(grokClaims(), payload, grokDeps())
).rejects.toMatchObject({
code: 'payload_too_large',
message: 'proxy envelope exceeds maxVisualRequestBodyBytes',
})
expect(signed).toBe(true)
expect((await counts()).rows[0]).toEqual(before)
expect((await readExisting()).rows[0]).toEqual(existingBefore)
if (!presented) {
expect(observedReservationId).toBeDefined()
expect(
(
await pool.query('SELECT id FROM budget_pending_reservations WHERE id = $1', [
observedReservationId,
])
).rows
).toHaveLength(0)
}
const leftover = await pool.query(
'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
[invocationId]
)
expect(leftover.rows).toHaveLength(0)
// The refusal must not burn the invocation/attempt identity: the
// same binding with an untouched ticket authorizes and commits.
grokTicketTap.afterIssue = null
const retried = await authorizeLlmProviderAttempt(grokClaims(), payload, grokDeps())
const committed = await pool.query(
'SELECT id, provider FROM llm_provider_attempts WHERE invocation_id = $1',
[invocationId]
)
expect(committed.rows).toEqual([
{ id: retried.providerAttemptId, provider: 'grok-subscription' },
])
expect((await readExisting()).rows[0]).toEqual(existingBefore)
} finally {
grokTicketTap.afterIssue = null
await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
__resetBudgetCheckCache()
}
}
)
})
})