Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
44.0 KB
Raw Blame
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import { randomBytes, randomUUID } from 'node:crypto'
import { once } from 'node:events'
import { readFileSync } from 'node:fs'
import { createServer as createHttpServer } from 'node:http'
import type { Socket } from 'node:net'
import { setImmediate as nextTurn } from 'node:timers/promises'
import { Pool } from 'pg'
import {
  LIMITS as GROK_LIMITS,
  computeGrokPolicyHash,
} from '@clerum/grok-provider-attempt-contract'
import { LIMITS } from '@clerum/llm-provider-attempt-contract'
import { config } from '../src/config.js'
import { type DbClient, initDb, withTransaction as withProductionTransaction } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import { __resetBudgetCheckCache, evaluateBudgetCheck } from '../src/services/budgets/check.js'
import { getActiveReservation } from '../src/services/budgets/reservations.js'
import { getCodexCatalogModelState } from '../src/services/codexSubscriptionCatalog.js'
import {
  getSafeCodexSubscriptionConnection,
  insertInitialCodexSubscriptionConnection,
  recordCodexCatalogOutcome,
} from '../src/services/codexSubscriptionConnection.js'
import type {
  IssuedGrokExecutionTicket,
  issueRegisteredGrokExecutionTicket,
} from '../src/services/grokProviderAttemptTicket.js'
import {
  getSafeGrokSubscriptionConnection,
  insertInitialGrokSubscriptionConnection,
  recordGrokCatalogOutcome,
} from '../src/services/grokSubscriptionConnection.js'
import {
  type LlmProviderAttemptAuthorizerDeps,
  authorizeLlmProviderAttempt,
  computeCodexPolicyHash,
} from '../src/services/llmProviderAttemptAuthorizer.js'
import { CODEX_ATTEMPT_RESERVATION_TTL_SECONDS } from '../src/services/llmProviderAttemptEnvelope.js'
import {
  getMaxLlmProviderAttemptGeneration,
  insertLlmProviderAttempt,
} from '../src/services/llmProviderAttemptStore.js'
import { issueRegisteredCodexExecutionTicket } from '../src/services/llmProviderAttemptTicket.js'
import { ResourceService } from '../src/services/resourceService.js'
import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
import { createPostgresCommitReplyBlackhole } from './helpers/realPostgresCancellation.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

// The Grok authorizer signs through this module, not through a dependency, so
// the Grok envelope test taps the real issuer here: the ticket is registered
// for real, then `afterIssue` may replace the returned value.
type IssueGrokTicket = typeof issueRegisteredGrokExecutionTicket
const grokTicketTap = vi.hoisted(() => ({
  afterIssue: null as
    | null
    | ((
        db: Parameters<IssueGrokTicket>[0],
        input: Parameters<IssueGrokTicket>[1],
        issued: IssuedGrokExecutionTicket
      ) => Promise<IssuedGrokExecutionTicket>),
}))
vi.mock('../src/services/grokProviderAttemptTicket.js', async importOriginal => {
  const actual =
    await importOriginal<typeof import('../src/services/grokProviderAttemptTicket.js')>()
  return {
    ...actual,
    issueRegisteredGrokExecutionTicket: async (...args: Parameters<IssueGrokTicket>) => {
      const issued = await actual.issueRegisteredGrokExecutionTicket(...args)
      return grokTicketTap.afterIssue ? grokTicketTap.afterIssue(args[0], args[1], issued) : issued
    },
  }
})

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

async function cancellationDeadline<T>(promise: Promise<T>): Promise<T> {
  let timer: NodeJS.Timeout | undefined
  try {
    return await Promise.race([
      promise,
      new Promise<never>((_, reject) => {
        timer = setTimeout(
          () => reject(new Error('authorizer cancellation phase did not complete')),
          2_000
        )
      }),
    ])
  } finally {
    clearTimeout(timer)
  }
}

async function observeCancellation(check: () => Promise<boolean>): Promise<void> {
  const deadline = Date.now() + 2_000
  while (!(await check())) {
    if (Date.now() >= deadline) throw new Error('authorizer physical state did not settle')
    await nextTurn()
  }
}

async function createAssignmentLookupPeer(stall: boolean) {
  let arrived!: () => void
  const arrival = new Promise<void>(resolve => {
    arrived = resolve
  })
  const sockets = new Set<Socket>()
  const server = createHttpServer((req, res) => {
    arrived()
    if (stall) return
    res.setHeader('Content-Type', 'application/json')
    res.end(JSON.stringify({ metadata: { name: 'research-host' } }))
  })
  server.on('connection', socket => {
    sockets.add(socket)
    socket.once('close', () => sockets.delete(socket))
  })
  server.listen(0, '127.0.0.1')
  await once(server, 'listening')
  const address = server.address()
  if (!address || typeof address === 'string') throw new Error('missing assignment peer address')
  const kubeconfig = new k8s.KubeConfig()
  kubeconfig.loadFromOptions({
    clusters: [
      // client-node requires an explicit opt-in for this owned loopback HTTP peer.
      { name: 'fixture', server: `http://127.0.0.1:${address.port}`, skipTLSVerify: true },
    ],
    users: [{ name: 'fixture' }],
    contexts: [{ name: 'fixture', cluster: 'fixture', user: 'fixture' }],
    currentContext: 'fixture',
  })
  return {
    resources: new ResourceService(kubeconfig.makeApiClient(k8s.CustomObjectsApi), 'fixture', {
      hosts: 'fixture',
    }),
    arrived: arrival,
    get openSocketCount() {
      return sockets.size
    },
    recover: () => {
      stall = false
    },
    close: async () => {
      for (const socket of sockets) socket.destroy()
      await new Promise<void>((resolve, reject) =>
        server.close(error => (error ? reject(error) : resolve()))
      )
    },
  }
}

const REQUEST = {
  schemaVersion: 'codex-completion-request.v1' as const,
  requestId: 'req-pg-1',
  idempotencyKey: 'idem-pg-1',
  provider: 'codex-subscription' as const,
  model: 'gpt-5.1',
  messages: [{ role: 'user' as const, content: 'hello from postgres' }],
}

function claims(): McpHostAccessClaims {
  return {
    sub: 'default/research-host',
    recipeNamespace: 'default',
    recipeName: 'research-host',
    hostRefs: ['research-host'],
    scope: 'workflow:approval:request',
    workflowControlScopes: ['llm:codex:execute'],
    mcpCapabilities: [],
    iss: 'control-api',
    aud: 'workflow-approvals',
    jti: randomUUID(),
    exp: Math.floor(Date.now() / 1000) + 60,
  }
}

describeRealPostgres('Codex provider-attempt authorization on real PostgreSQL', () => {
  const database = `llm_attempt_authz_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? `postgresql://postgres@${['127', '0', '0', '1'].join('.')}/postgres`,
    database
  )
  let adminPool: Pool
  let pool: Pool

  function dbClient() {
    return { query: (text: string, values?: unknown[]) => pool.query(text, values) }
  }

  function testDeps(
    overrides: Partial<LlmProviderAttemptAuthorizerDeps> = {}
  ): LlmProviderAttemptAuthorizerDeps {
    const db = dbClient()
    return {
      enabled: true,
      db,
      withTransaction: async work => {
        const client = await pool.connect()
        try {
          await client.query('BEGIN')
          const result = await work(client as never)
          await client.query('COMMIT')
          return result
        } catch (error) {
          try {
            await client.query('ROLLBACK')
          } catch {
            // keep the original error
          }
          throw error
        } finally {
          client.release()
        }
      },
      getConnection: getSafeCodexSubscriptionConnection,
      getModelState: getCodexCatalogModelState,
      resolveAssignment: async () => ({
        liveBrokerProviders: ['codex-subscription'],
        liveConnectionRef: 'deployment-default',
      }),
      evaluateBudget: evaluateBudgetCheck,
      getActiveReservation,
      getMaxGeneration: getMaxLlmProviderAttemptGeneration,
      insertAttempt: insertLlmProviderAttempt,
      issueTicket: issueRegisteredCodexExecutionTicket,
      ...overrides,
    }
  }

  function registerProductionCancellationCases(fixture: {
    provider: 'codex-subscription' | 'grok-subscription'
    model: string
    connectionKey: string
    caller: () => McpHostAccessClaims
    request: () => Record<string, unknown>
    readConnection: () => Promise<{
      id: string
      catalogRevision: number
      credentialRevision: number
      connectionKey: string
    } | null>
    policyHash: (connection: {
      catalogRevision: number
      credentialRevision: number
      connectionKey: string
    }) => string
  }): void {
    describe(`${fixture.provider} production cancellation`, () => {
      async function run(stage: 'assignment' | 'query' | 'commit'): Promise<void> {
        const invocationId = `actual-cancel-${randomUUID()}`
        const budgetId = randomUUID()
        const current = await fixture.readConnection()
        expect(current).not.toBeNull()
        const payload = {
          request: fixture.request(),
          invocationId,
          attemptGeneration: 1,
          providerAttemptIndex: 1,
          policyRevision: current!.catalogRevision,
          policyHash: fixture.policyHash(current!),
        }
        const observer = new Pool({
          connectionString,
          max: 1,
          connectionTimeoutMillis: 2_000,
          statement_timeout: 2_000,
          idleTimeoutMillis: 1_000,
        })
        const proxy =
          stage === 'commit'
            ? await createPostgresCommitReplyBlackhole(connectionString)
            : undefined
        const workPool = new Pool({
          connectionString: proxy?.connectionString ?? connectionString,
          ssl: false,
          max: 1,
          connectionTimeoutMillis: 2_000,
          statement_timeout: 15_000,
          idleTimeoutMillis: 1_000,
        })
        const lookup = await createAssignmentLookupPeer(stage === 'assignment')
        const controller = new AbortController()
        const reason = Object.assign(new Error('authorize_timeout'), { code: 'authorize_timeout' })
        const phases: string[] = []
        let backendPid: number | undefined
        let attemptId: string | undefined
        let reservationId: string | undefined
        let stallQuery = stage === 'query'
        let settled = false
        const previousTap = grokTicketTap.afterIssue

        async function counts() {
          return (
            await observer.query<{ attempts: number; tickets: number; reservations: number }>(
              `SELECT
             (SELECT COUNT(*)::int FROM llm_provider_attempts WHERE invocation_id = $1) AS attempts,
             (SELECT COUNT(*)::int FROM llm_provider_attempt_tickets t JOIN llm_provider_attempts a
               ON t.provider_attempt_id = a.id WHERE a.invocation_id = $1) AS tickets,
             (SELECT COUNT(*)::int FROM budget_pending_reservations WHERE budget_id = $2) AS reservations`,
              [invocationId, budgetId]
            )
          ).rows[0]
        }

        async function afterRealIssue(
          db: DbClient,
          input: { providerAttemptId: string; budgetReservationId: string },
          issued: { claims: { jti: string } }
        ): Promise<void> {
          const registered = await db.query(
            `SELECT a.provider, a.connection_id, a.budget_reservation_id,
                    t.jti, t.status AS ticket_status, r.budget_id, r.host_ref
               FROM llm_provider_attempts a
               JOIN llm_provider_attempt_tickets t ON t.provider_attempt_id = a.id
               JOIN budget_pending_reservations r ON r.id::text = a.budget_reservation_id
              WHERE a.id = $1`,
            [input.providerAttemptId]
          )
          expect(registered.rows).toEqual([
            {
              provider: fixture.provider,
              connection_id: current!.id,
              budget_reservation_id: input.budgetReservationId,
              jti: issued.claims.jti,
              ticket_status: 'issued',
              budget_id: budgetId,
              host_ref: fixture.caller().hostRefs[0],
            },
          ])
          attemptId = input.providerAttemptId
          reservationId = input.budgetReservationId
          phases.push('ticket_registered')
          // The real issuer has already signed and registered its ticket. This
          // native statement supplies a deterministic interruptible phase; no
          // test barrier resolves the work after cancellation.
          if (stallQuery) await db.query('SELECT pg_sleep(30)')
        }

        function productionDeps(signal: AbortSignal): LlmProviderAttemptAuthorizerDeps {
          if (fixture.provider === 'grok-subscription') {
            grokTicketTap.afterIssue = async (db, input, issued) => {
              await afterRealIssue(db, input, issued)
              return issued
            }
          }
          return testDeps({
            signal,
            resolveAssignment: async (_hostRef, forwarded) => {
              expect(forwarded).toBe(signal)
              phases.push('assignment')
              await lookup.resources.getResource('hosts', 'research-host', 'fixture', forwarded)
              return {
                liveBrokerProviders: [fixture.provider],
                liveConnectionRef: fixture.connectionKey,
              }
            },
            withTransaction: async (work, _unused, options) => {
              // Assert the authorizer's actual forwarding before binding this
              // fixture database; do not inject a separate cancellation source.
              expect(options?.signal).toBe(signal)
              return withProductionTransaction(
                async tx => {
                  phases.push('transaction')
                  const identity = await tx.query('SELECT pg_backend_pid() AS pid')
                  backendPid = (identity.rows[0] as { pid: number }).pid
                  return work(tx)
                },
                workPool,
                options
              )
            },
            evaluateBudget: async (...args) => {
              const result = await evaluateBudgetCheck(...args)
              expect(result.allowed).toBe(true)
              expect(result.reservationIds).toHaveLength(1)
              phases.push('budget_reserved')
              return result
            },
            insertAttempt: async (db, input) => {
              const attempt = await insertLlmProviderAttempt(db, input)
              phases.push('attempt_inserted')
              return attempt
            },
            issueTicket: async (db, input) => {
              const issued = await issueRegisteredCodexExecutionTicket(db, input)
              await afterRealIssue(db, input, issued)
              return issued
            },
          })
        }

        try {
          const runtime = (
            await observer.query<{ version: string; version_num: string }>(
              "SELECT version(), current_setting('server_version_num') AS version_num"
            )
          ).rows[0]
          expect(Number(runtime.version_num)).toBeGreaterThanOrEqual(160000)
          expect(Number(runtime.version_num)).toBeLessThan(170000)
          expect(runtime.version).toMatch(/linux|darwin|bsd|illumos/i)
          await observer.query(
            `INSERT INTO token_budgets
              (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
             VALUES ($1, 'actual authorize cancellation', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
            [budgetId, JSON.stringify({ provider: [fixture.provider], model: [fixture.model] })]
          )
          __resetBudgetCheckCache()
          const outcome = authorizeLlmProviderAttempt(
            fixture.caller(),
            payload,
            productionDeps(controller.signal)
          )
            .then(
              value => ({ value }),
              error => ({ error })
            )
            .finally(() => {
              settled = true
            })
          async function beforeCancellation<T>(phase: Promise<T>): Promise<T> {
            // Report an actual authorization failure instead of timing out on a
            // phase that the rejected production path can no longer reach.
            const unexpectedCompletion = outcome.then(result => {
              if ('error' in result) throw result.error
              throw new Error('authorizer completed before its cancellation phase')
            })
            if (settled) return unexpectedCompletion
            return cancellationDeadline(Promise.race([phase, unexpectedCompletion]))
          }
          const expectedPhases = [
            'assignment',
            'transaction',
            'budget_reserved',
            'attempt_inserted',
            'ticket_registered',
          ]
          if (stage === 'assignment') {
            await beforeCancellation(lookup.arrived)
            expect(phases).toEqual(['assignment'])
            expect(workPool.totalCount).toBe(0)
            expect(lookup.openSocketCount).toBe(1)
          } else if (stage === 'query') {
            await observeCancellation(async () => {
              if (!backendPid) return beforeCancellation(Promise.resolve(false))
              const result = await beforeCancellation(
                observer.query<{ state: string; query: string }>(
                  'SELECT state, query FROM pg_stat_activity WHERE pid = $1',
                  [backendPid]
                )
              )
              return (
                result.rows[0]?.state === 'active' &&
                result.rows[0]?.query === 'SELECT pg_sleep(30)'
              )
            })
            expect(phases).toEqual(expectedPhases)
            expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
          } else {
            await beforeCancellation(proxy!.commitForwarded)
            await observeCancellation(
              async () => (await beforeCancellation(counts())).attempts === 1
            )
            await observeCancellation(async () => proxy!.discardedReplyBytes > 0)
            expect(phases).toEqual(expectedPhases)
            expect(proxy!.commands).toEqual(['BEGIN', 'COMMIT'])
            expect(proxy!.discardedReplyBytes).toBeGreaterThan(0)
          }
          expect(settled).toBe(false)
          controller.abort(reason)
          const interrupted = await cancellationDeadline(outcome)
          expect('error' in interrupted ? interrupted.error : undefined).toBe(reason)
          expect(workPool.totalCount).toBe(0)
          if (stage === 'assignment') {
            await observeCancellation(async () => lookup.openSocketCount === 0)
          }
          if (backendPid) {
            await observeCancellation(
              async () =>
                (
                  await observer.query<{ count: number }>(
                    'SELECT COUNT(*)::int AS count FROM pg_stat_activity WHERE pid = $1',
                    [backendPid]
                  )
                ).rows[0].count === 0
            )
          }
          await withProductionTransaction(async db => {
            const result = await db.query(
              'SELECT pg_try_advisory_xact_lock(hashtext($1)) AS available',
              [budgetId]
            )
            expect((result.rows[0] as { available: boolean }).available).toBe(true)
          }, observer)
          if (stage === 'commit') {
            // The client never received COMMIT acknowledgement, yet the real
            // business ledger is durable. Do not label this rollback/no spend.
            expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
            const attempt = await observer.query(
              'SELECT id, provider, connection_id, budget_reservation_id, status FROM llm_provider_attempts WHERE invocation_id = $1',
              [invocationId]
            )
            expect(attempt.rows).toEqual([
              {
                id: attemptId,
                provider: fixture.provider,
                connection_id: current!.id,
                budget_reservation_id: reservationId,
                status: 'authorized',
              },
            ])
          } else {
            expect(await counts()).toEqual({ attempts: 0, tickets: 0, reservations: 0 })
            stallQuery = false
            lookup.recover()
            phases.length = 0
            const later = await authorizeLlmProviderAttempt(
              fixture.caller(),
              payload,
              productionDeps(new AbortController().signal)
            )
            expect(later.providerAttemptId).toBe(attemptId)
            expect(phases).toEqual(expectedPhases)
            expect(await counts()).toEqual({ attempts: 1, tickets: 1, reservations: 1 })
            expect(workPool.idleCount).toBe(1)
          }
        } finally {
          controller.abort(reason)
          grokTicketTap.afterIssue = previousTap
          proxy?.allowReplies()
          await endPoolAndWaitForClients(workPool)
          await proxy?.close()
          await lookup.close()
          await observer.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
          __resetBudgetCheckCache()
          await endPoolAndWaitForClients(observer)
        }
      }

      it(
        'cancels a configured Kubernetes lookup before any production transaction, then reauthorizes the same binding',
        () => run('assignment'),
        10_000
      )
      it(
        'cancels a native query after real budget, attempt and signed ticket writes, then rolls back and reauthorizes',
        () => run('query'),
        10_000
      )
      it(
        'keeps real authorization ledger rows when COMMIT executes but its reply is lost',
        () => run('commit'),
        10_000
      )
    })
  }

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    pool = new Pool({ connectionString })
    await initDb({ connect: () => pool.connect() })
    const created = await insertInitialCodexSubscriptionConnection(pool, KEY, {
      refreshToken: 'refresh-authz',
      accountFingerprint: 'fp-authz',
    })
    await recordCodexCatalogOutcome(pool, {
      catalogStatus: 'ready',
      expectedCredentialRevision: 1,
      expectedCatalogRevision: 0,
    })
    await pool.query(
      `INSERT INTO codex_catalog_models
         (connection_id, model, enabled, source, discovered_at, last_seen_at, stale)
       VALUES ($1, 'gpt-5.1', true, 'discovery', NOW(), NOW(), false)`,
      [created.id]
    )
  }, 60_000)

  afterAll(async () => {
    try {
      await pool?.end()
      if (adminPool) {
        await adminPool
          .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
          .catch(() => undefined)
      }
    } finally {
      await adminPool?.end()
    }
  })

  it('lets only one concurrent authorization win for the same invocation binding', async () => {
    const current = await getSafeCodexSubscriptionConnection(pool)
    expect(current?.status).toBe('connected')
    const payload = {
      request: REQUEST,
      invocationId: `invocation-${randomUUID()}`,
      attemptGeneration: 1,
      providerAttemptIndex: 1,
      policyRevision: current!.catalogRevision,
      policyHash: computeCodexPolicyHash({
        model: REQUEST.model,
        catalogRevision: current!.catalogRevision,
        credentialRevision: current!.credentialRevision,
        connectionKey: current!.connectionKey,
      }),
    }
    const deps = testDeps()
    const [first, second] = await Promise.allSettled([
      authorizeLlmProviderAttempt(claims(), payload, deps),
      authorizeLlmProviderAttempt(claims(), payload, deps),
    ])
    const wins = [first, second].filter(result => result.status === 'fulfilled')
    const losses = [first, second].filter(result => result.status === 'rejected')
    expect(wins).toHaveLength(1)
    expect(losses).toHaveLength(1)
    if (losses[0]?.status === 'rejected') {
      expect((losses[0].reason as { code?: string }).code).toBe('idempotency_conflict')
    }

    const count = await pool.query<{ count: string }>(
      `SELECT COUNT(*)::text AS count
         FROM llm_provider_attempts
        WHERE invocation_id = $1`,
      [payload.invocationId]
    )
    expect(count.rows[0]?.count).toBe('1')
    expect(JSON.stringify(wins[0])).not.toContain('hello from postgres')
  })

  it('rolls budget reservation, attempt, and ticket back together when ticket issue fails', async () => {
    const current = await getSafeCodexSubscriptionConnection(pool)
    const invocationId = `invocation-rollback-${randomUUID()}`
    const payload = {
      request: { ...REQUEST, requestId: 'req-pg-2', idempotencyKey: 'idem-pg-2' },
      invocationId,
      attemptGeneration: 1,
      providerAttemptIndex: 1,
      policyRevision: current!.catalogRevision,
      policyHash: computeCodexPolicyHash({
        model: REQUEST.model,
        catalogRevision: current!.catalogRevision,
        credentialRevision: current!.credentialRevision,
        connectionKey: current!.connectionKey,
      }),
    }

    await expect(
      authorizeLlmProviderAttempt(
        claims(),
        payload,
        testDeps({
          issueTicket: async () => {
            throw new Error('ticket registration failed')
          },
        })
      )
    ).rejects.toThrow(/ticket registration failed/)

    const leftover = await pool.query<{ count: string }>(
      `SELECT COUNT(*)::text AS count
         FROM llm_provider_attempts
        WHERE invocation_id = $1`,
      [invocationId]
    )
    expect(leftover.rows[0]?.count).toBe('0')
  })

  it('reserves a danger-zone token budget for the whole attempt lifetime', async () => {
    const current = await getSafeCodexSubscriptionConnection(pool)
    // remaining (1000) < max_task_amount (2000) puts the budget in the danger
    // zone, and min_start 0 lets the attempt through with a reservation.
    const budget = await pool.query<{ id: string }>(
      `INSERT INTO token_budgets
         (name, scope, unit, limit_amount, period, timezone,
          min_start_amount, max_task_amount, enforcement)
       VALUES ('attempt-envelope', '{}'::jsonb, 'tokens', 1000, 'monthly', 'UTC', 0, 2000, 'block')
       RETURNING id`
    )
    const budgetId = budget.rows[0]!.id
    __resetBudgetCheckCache()
    try {
      const invocationId = `invocation-envelope-${randomUUID()}`
      const authorized = await authorizeLlmProviderAttempt(
        claims(),
        {
          request: { ...REQUEST, requestId: 'req-pg-3', idempotencyKey: 'idem-pg-3' },
          invocationId,
          attemptGeneration: 1,
          providerAttemptIndex: 1,
          policyRevision: current!.catalogRevision,
          policyHash: computeCodexPolicyHash({
            model: REQUEST.model,
            catalogRevision: current!.catalogRevision,
            credentialRevision: current!.credentialRevision,
            connectionKey: current!.connectionKey,
          }),
        },
        testDeps()
      )
      expect(authorized.providerAttemptId).toEqual(expect.any(String))

      // created_at and expires_at come from the same statement's NOW(), so
      // their difference is exactly the TTL the reservation was written with.
      const reservations = await pool.query<{
        id: string
        task_ref: string
        ttl_seconds: number
      }>(
        `SELECT id, task_ref,
                EXTRACT(EPOCH FROM (expires_at - created_at))::int AS ttl_seconds
           FROM budget_pending_reservations
          WHERE budget_id = $1`,
        [budgetId]
      )
      expect(reservations.rows).toEqual([
        {
          id: expect.any(String),
          task_ref: `${invocationId}:1:1`,
          ttl_seconds: CODEX_ATTEMPT_RESERVATION_TTL_SECONDS,
        },
      ])
      expect(CODEX_ATTEMPT_RESERVATION_TTL_SECONDS).toBe(2160)

      const attempt = await pool.query<{ budget_reservation_id: string }>(
        `SELECT budget_reservation_id FROM llm_provider_attempts WHERE invocation_id = $1`,
        [invocationId]
      )
      expect(attempt.rows).toEqual([{ budget_reservation_id: reservations.rows[0]!.id }])
    } finally {
      await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
      __resetBudgetCheckCache()
    }
  })

  it.each([false, true])(
    'rolls back an oversized signed V2 envelope without losing an existing reservation (presented=%s)',
    async presented => {
      const current = await getSafeCodexSubscriptionConnection(pool)
      const invocationId = `visual-envelope-${randomUUID()}`
      const request = {
        ...REQUEST,
        schemaVersion: 'codex-completion-request.v2',
        messages: [{ role: 'user', content: '' }],
      }
      const budgetId = randomUUID()
      const existingReservationId = randomUUID()
      // Force the real danger-zone path without relying on rollup timing. The
      // configured maximum task is greater than the remaining period allowance.
      await pool.query(
        `INSERT INTO token_budgets
        (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
       VALUES ($1, 'visual envelope rollback', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
        [budgetId, JSON.stringify({ provider: ['codex-subscription'], model: [REQUEST.model] })]
      )
      await pool.query(
        `INSERT INTO budget_pending_reservations
        (id, budget_id, est_amount, task_ref, host_ref, expires_at)
       VALUES ($1, $2, 5, 'preexisting-visual-task', 'research-host', NOW() + INTERVAL '15 minutes')`,
        [existingReservationId, budgetId]
      )
      __resetBudgetCheckCache()
      const payload = {
        request,
        invocationId,
        attemptGeneration: 1,
        providerAttemptIndex: 1,
        policyRevision: current!.catalogRevision,
        policyHash: computeCodexPolicyHash({
          model: REQUEST.model,
          catalogRevision: current!.catalogRevision,
          credentialRevision: current!.credentialRevision,
          connectionKey: current!.connectionKey,
        }),
        ...(presented ? { budgetReservationId: existingReservationId } : {}),
      }
      request.messages[0].content = 'x'.repeat(
        LIMITS.maxRequestBodyBytes - Buffer.byteLength(JSON.stringify(payload)) - 16
      )
      const counts = () =>
        pool.query(`SELECT
      (SELECT count(*)::text FROM llm_provider_attempts) AS attempts,
      (SELECT count(*)::text FROM llm_provider_attempt_tickets) AS tickets,
      (SELECT count(*)::text FROM budget_pending_reservations) AS reservations`)
      const readExisting = () =>
        pool.query('SELECT * FROM budget_pending_reservations WHERE id = $1', [
          existingReservationId,
        ])
      const before = (await counts()).rows[0]
      const existingBefore = (await readExisting()).rows[0]
      let observedReservationId: string | undefined
      let signed = false
      try {
        await expect(
          authorizeLlmProviderAttempt(
            claims(),
            payload,
            testDeps({
              issueTicket: async (db, input) => {
                observedReservationId = input.budgetReservationId
                const active = await db.query(
                  'SELECT id FROM budget_pending_reservations WHERE budget_id = $1',
                  [budgetId]
                )
                expect(active.rows).toHaveLength(presented ? 1 : 2)
                if (presented) expect(input.budgetReservationId).toBe(existingReservationId)
                else {
                  expect(input.budgetReservationId).not.toBe('unbudgeted')
                  expect(input.budgetReservationId).not.toBe(existingReservationId)
                }
                const issued = await issueRegisteredCodexExecutionTicket(db, input)
                const registered = await db.query(
                  'SELECT jti FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1',
                  [input.providerAttemptId]
                )
                expect(registered.rows).toHaveLength(1)
                signed = true
                // The request's text budget stays at 1 MiB. Inject an oversized
                // return value only after a real ticket/reservation exists in
                // this transaction, to prove the final envelope guard rolls
                // back database writes. This value never leaves the test.
                return {
                  ...issued,
                  executionTicket:
                    issued.executionTicket + 'x'.repeat(LIMITS.maxVisualRequestBodyBytes),
                }
              },
            })
          )
        ).rejects.toMatchObject({ code: 'payload_too_large' })
        expect(signed).toBe(true)
        expect((await counts()).rows[0]).toEqual(before)
        expect((await readExisting()).rows[0]).toEqual(existingBefore)
        if (!presented) {
          expect(observedReservationId).toBeDefined()
          expect(
            (
              await pool.query('SELECT id FROM budget_pending_reservations WHERE id = $1', [
                observedReservationId,
              ])
            ).rows
          ).toHaveLength(0)
        }
        const leftover = await pool.query(
          'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
          [invocationId]
        )
        expect(leftover.rows).toHaveLength(0)
        // A failed size check must not burn the invocation/attempt identity.
        // Retrying that same binding with a request that fits must authorize.
        request.messages[0].content = 'A bounded request after rollback'
        const retried = await authorizeLlmProviderAttempt(claims(), payload, testDeps())
        const committed = await pool.query(
          'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
          [invocationId]
        )
        expect(committed.rows).toEqual([{ id: retried.providerAttemptId }])
        expect((await readExisting()).rows[0]).toEqual(existingBefore)
      } finally {
        await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
        __resetBudgetCheckCache()
      }
    }
  )

  registerProductionCancellationCases({
    provider: 'codex-subscription',
    model: REQUEST.model,
    connectionKey: 'deployment-default',
    caller: claims,
    readConnection: () => getSafeCodexSubscriptionConnection(pool),
    request: () => ({
      ...REQUEST,
      schemaVersion: 'codex-completion-request.v2',
      requestId: randomUUID(),
      idempotencyKey: randomUUID(),
    }),
    policyHash: connection =>
      computeCodexPolicyHash({
        model: REQUEST.model,
        catalogRevision: connection.catalogRevision,
        credentialRevision: connection.credentialRevision,
        connectionKey: connection.connectionKey,
      }),
  })

  // #784: the Grok twin of the Codex rollback above, on an image-bearing V2
  // request. The Grok path checks its proxy envelope after signing and before
  // commit, so a refusal there must roll back the attempt, the registered
  // ticket and any new reservation.
  describe('Grok V2 proxy envelope', () => {
    const GROK_CONNECTION_KEY = 'team-grok-visual'
    const GROK_MODEL = 'grok-4.6'
    const previousGrokFlag = config.grokSubscriptionEnabled

    function grokClaims(): McpHostAccessClaims {
      return { ...claims(), workflowControlScopes: ['llm:grok:execute'] }
    }

    /** The real 2x2 PNG part of the Grok contract's V2 fixture. */
    function fixturePngPart(): unknown {
      const fixture = JSON.parse(
        readFileSync(
          new URL(
            '../../packages/grok-provider-attempt-contract/fixtures/visual-requests.json',
            import.meta.url
          ),
          'utf8'
        )
      ) as { png: { messages: Array<{ contentParts: Array<{ type: string }> }> } }
      const part = fixture.png.messages[0]!.contentParts.find(item => item.type === 'image')
      expect(part).toBeDefined()
      return part
    }

    beforeAll(async () => {
      const created = await insertInitialGrokSubscriptionConnection(
        pool,
        KEY,
        { refreshToken: 'refresh-grok-visual', accountFingerprint: 'fp-grok-visual' },
        GROK_CONNECTION_KEY
      )
      await recordGrokCatalogOutcome(pool, {
        catalogStatus: 'ready',
        connectionStatus: 'connected',
        expectedCredentialRevision: 1,
        expectedCatalogRevision: 0,
        connectionKey: GROK_CONNECTION_KEY,
      })
      await pool.query(
        `INSERT INTO grok_catalog_models
           (connection_id, model, enabled, source, discovered_at, last_seen_at, stale)
         VALUES ($1, $2, true, 'discovery', NOW(), NOW(), false)`,
        [created.id, GROK_MODEL]
      )
    })

    // Per-test flag and tap state: the file's only afterAll owns the
    // administrative pool close (realPostgresAdminCleanup.guard.test.ts).
    beforeEach(() => {
      config.grokSubscriptionEnabled = true
    })

    afterEach(() => {
      config.grokSubscriptionEnabled = previousGrokFlag
      grokTicketTap.afterIssue = null
    })

    registerProductionCancellationCases({
      provider: 'grok-subscription',
      model: GROK_MODEL,
      connectionKey: GROK_CONNECTION_KEY,
      caller: grokClaims,
      readConnection: () => getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY),
      request: () => ({
        schemaVersion: 'grok-completion-request.v2',
        requestId: randomUUID(),
        idempotencyKey: randomUUID(),
        provider: 'grok-subscription',
        model: GROK_MODEL,
        messages: [
          {
            role: 'user',
            content: 'look',
            contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],
          },
        ],
      }),
      policyHash: connection =>
        computeGrokPolicyHash({
          model: GROK_MODEL,
          catalogRevision: connection.catalogRevision,
          credentialRevision: connection.credentialRevision,
          connectionKey: GROK_CONNECTION_KEY,
        }),
    })

    it.each([false, true])(
      'rolls back an oversized signed Grok V2 envelope without losing an existing reservation (presented=%s)',
      async presented => {
        const current = await getSafeGrokSubscriptionConnection(pool, GROK_CONNECTION_KEY)
        expect(current?.status).toBe('connected')
        const invocationId = `grok-visual-envelope-${randomUUID()}`
        const request = {
          schemaVersion: 'grok-completion-request.v2',
          requestId: `req-${invocationId}`,
          idempotencyKey: `idem-${invocationId}`,
          provider: 'grok-subscription',
          model: GROK_MODEL,
          messages: [
            {
              role: 'user',
              content: 'look',
              contentParts: [{ type: 'text', text: 'look' }, fixturePngPart()],
            },
          ],
        }
        const budgetId = randomUUID()
        const existingReservationId = randomUUID()
        await pool.query(
          `INSERT INTO token_budgets
          (id, name, scope, unit, limit_amount, period, min_start_amount, max_task_amount, enforcement)
         VALUES ($1, 'grok visual envelope rollback', $2::jsonb, 'tokens', 100, 'daily', 1, 200, 'block')`,
          [budgetId, JSON.stringify({ provider: ['grok-subscription'], model: [GROK_MODEL] })]
        )
        await pool.query(
          `INSERT INTO budget_pending_reservations
          (id, budget_id, est_amount, task_ref, host_ref, expires_at)
         VALUES ($1, $2, 5, 'preexisting-grok-visual-task', 'research-host', NOW() + INTERVAL '15 minutes')`,
          [existingReservationId, budgetId]
        )
        __resetBudgetCheckCache()
        const payload = {
          request,
          invocationId,
          attemptGeneration: 1,
          providerAttemptIndex: 1,
          policyRevision: current!.catalogRevision,
          policyHash: computeGrokPolicyHash({
            model: GROK_MODEL,
            catalogRevision: current!.catalogRevision,
            credentialRevision: current!.credentialRevision,
            connectionKey: GROK_CONNECTION_KEY,
          }),
          ...(presented ? { budgetReservationId: existingReservationId } : {}),
        }
        const grokDeps = () =>
          testDeps({
            resolveAssignment: async () => ({
              liveBrokerProviders: ['grok-subscription'],
              liveConnectionRef: GROK_CONNECTION_KEY,
            }),
          })
        const counts = () =>
          pool.query(`SELECT
        (SELECT count(*)::text FROM llm_provider_attempts) AS attempts,
        (SELECT count(*)::text FROM llm_provider_attempt_tickets) AS tickets,
        (SELECT count(*)::text FROM budget_pending_reservations) AS reservations`)
        const readExisting = () =>
          pool.query('SELECT * FROM budget_pending_reservations WHERE id = $1', [
            existingReservationId,
          ])
        const before = (await counts()).rows[0]
        const existingBefore = (await readExisting()).rows[0]
        let observedReservationId: string | undefined
        let signed = false
        grokTicketTap.afterIssue = async (db, input, issued) => {
          observedReservationId = input.budgetReservationId
          const active = await db.query(
            'SELECT id FROM budget_pending_reservations WHERE budget_id = $1',
            [budgetId]
          )
          expect(active.rows).toHaveLength(presented ? 1 : 2)
          if (presented) expect(input.budgetReservationId).toBe(existingReservationId)
          else {
            expect(input.budgetReservationId).not.toBe('unbudgeted')
            expect(input.budgetReservationId).not.toBe(existingReservationId)
          }
          const registered = await db.query(
            'SELECT jti FROM llm_provider_attempt_tickets WHERE provider_attempt_id = $1',
            [input.providerAttemptId]
          )
          expect(registered.rows).toEqual([{ jti: issued.claims.jti }])
          signed = true
          // No real image-bearing request can exceed the 35 MiB envelope once
          // its images and text are inside their own budgets. Inflate the
          // returned ticket only after the real ticket and reservation exist
          // in this transaction. This value never leaves the test.
          return {
            ...issued,
            executionTicket:
              issued.executionTicket + 'x'.repeat(GROK_LIMITS.maxVisualRequestBodyBytes),
          }
        }
        try {
          await expect(
            authorizeLlmProviderAttempt(grokClaims(), payload, grokDeps())
          ).rejects.toMatchObject({
            code: 'payload_too_large',
            message: 'proxy envelope exceeds maxVisualRequestBodyBytes',
          })
          expect(signed).toBe(true)
          expect((await counts()).rows[0]).toEqual(before)
          expect((await readExisting()).rows[0]).toEqual(existingBefore)
          if (!presented) {
            expect(observedReservationId).toBeDefined()
            expect(
              (
                await pool.query('SELECT id FROM budget_pending_reservations WHERE id = $1', [
                  observedReservationId,
                ])
              ).rows
            ).toHaveLength(0)
          }
          const leftover = await pool.query(
            'SELECT id FROM llm_provider_attempts WHERE invocation_id = $1',
            [invocationId]
          )
          expect(leftover.rows).toHaveLength(0)
          // The refusal must not burn the invocation/attempt identity: the
          // same binding with an untouched ticket authorizes and commits.
          grokTicketTap.afterIssue = null
          const retried = await authorizeLlmProviderAttempt(grokClaims(), payload, grokDeps())
          const committed = await pool.query(
            'SELECT id, provider FROM llm_provider_attempts WHERE invocation_id = $1',
            [invocationId]
          )
          expect(committed.rows).toEqual([
            { id: retried.providerAttemptId, provider: 'grok-subscription' },
          ])
          expect((await readExisting()).rows[0]).toEqual(existingBefore)
        } finally {
          grokTicketTap.afterIssue = null
          await pool.query('DELETE FROM token_budgets WHERE id = $1', [budgetId])
          __resetBudgetCheckCache()
        }
      }
    )
  })
})