Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
33.1 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool, type PoolClient } from 'pg'
import { initDb } from '../src/db.js'
import {
  CODEX_IN_FLIGHT_USAGE_GRACE_MS,
  insertLlmProviderAttempt,
} from '../src/services/llmProviderAttemptStore.js'
import { failStaleInvocationsInTransaction } from '../src/services/pluginWorkloadSdkDb.js'
import {
  PromptBridgeFinalizationError,
  type PromptBridgeFinalizationInput,
  finalizePromptBridgeInTransaction,
} from '../src/services/pluginWorkloadSdkFinalization.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

/**
 * Every transaction in this suite runs as the PRODUCTION runtime role, not as
 * the superuser that creates the database. `plugin_workload_sdk_spend_outcomes`
 * is granted SELECT/INSERT only (access profile `append`), so a write outside
 * that envelope raises 42501 and fails the test. Running these paths as a
 * superuser is exactly how an UPDATE against the spend ledger shipped green.
 *
 * `SET LOCAL` is transaction-scoped, so a pooled connection can never leak the
 * role into a later checkout, and `CREATE DATABASE` / `initDb` / `DROP
 * DATABASE` stay superuser.
 */
async function begin(client: PoolClient): Promise<void> {
  await client.query('BEGIN')
  await client.query('SET LOCAL ROLE control_api_runtime')
}

async function commit(client: PoolClient): Promise<void> {
  await client.query('COMMIT')
}

function deferred<T = void>(): {
  promise: Promise<T>
  resolve: (value: T | PromiseLike<T>) => void
} {
  let resolve!: (value: T | PromiseLike<T>) => void
  const promise = new Promise<T>(value => {
    resolve = value
  })
  return { promise, resolve }
}

describeRealPostgres('Plugin Workload SDK finalization on real PostgreSQL', () => {
  const database = `control_api_sdk_finalization_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? 'postgresql://[email protected]/postgres',
    database
  )
  let adminPool: Pool
  let dbPool: Pool

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)
    dbPool = new Pool({ connectionString })
    await initDb({ connect: () => dbPool.connect() })
  })

  afterAll(async () => {
    try {
      await endPoolAndWaitForClients(dbPool)
      if (adminPool) {
        await adminPool.query(
          `SELECT pg_terminate_backend(pid)
           FROM pg_stat_activity
          WHERE datname = $1
            AND pid <> pg_backend_pid()`,
          [database]
        )
        await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
      }
    } finally {
      await adminPool?.end()
    }
  })

  const CODEX_TARGET = {
    targetRef: 'primary-codex',
    provider: 'codex-subscription',
    model: 'gpt-5.1',
    credentialSlot: '',
  }
  const OPENAI_TARGET = {
    targetRef: 'primary-openai',
    provider: 'openai',
    model: 'gpt-4o-mini',
    credentialSlot: 'openai-api-key',
  }

  async function seedAttempt(
    client: PoolClient,
    options: {
      status?: PromptBridgeFinalizationInput['status']
      leaseExpired?: boolean
      codex?: boolean
    } = {}
  ): Promise<PromptBridgeFinalizationInput> {
    const invocationId = randomUUID()
    const providerAttemptId = randomUUID()
    const recipeNamespace = 'sandbox-recipes'
    const recipeName = `real-finalization-${invocationId}`
    const status = options.status ?? 'provider_unavailable'
    const leaseExpression = options.leaseExpired
      ? "now() - interval '5 minutes'"
      : "now() + interval '5 minutes'"
    const target = options.codex ? { ...CODEX_TARGET } : { ...OPENAI_TARGET }
    await client.query(
      `INSERT INTO plugin_workload_sdk_invocations
         (id, recipe_namespace, recipe_name, caller_ref, method, detail,
          idempotency_key_hash, payload_hash, status, authorization_decision,
          contract_version, attempt_generation, lease_expires_at)
       VALUES ($1, $2, $3, 'integration-test', 'promptBridge', '{}',
               $4, $5, 'in_progress', 'allowed', 2, 1, ${leaseExpression})`,
      [invocationId, recipeNamespace, recipeName, randomUUID(), randomUUID()]
    )
    await client.query(
      `INSERT INTO plugin_workload_sdk_invocation_attempts
         (invocation_id, recipe_namespace, recipe_name, attempt_generation,
          method, target_refs, status, lease_expires_at)
       VALUES ($1, $2, $3, 1, 'promptBridge', $4::jsonb, 'in_progress',
               ${leaseExpression})`,
      [invocationId, recipeNamespace, recipeName, JSON.stringify([target.targetRef])]
    )
    await client.query(
      `INSERT INTO plugin_workload_sdk_provider_attempts
         (id, invocation_id, recipe_namespace, recipe_name, attempt_generation,
          attempt_index, target_ref, provider, model, credential_slot, status,
          lease_expires_at)
       VALUES ($1, $2, $3, $4, 1, 1, $5, $6, $7, $8, 'in_progress',
               now() + interval '5 minutes')`,
      [
        providerAttemptId,
        invocationId,
        recipeNamespace,
        recipeName,
        target.targetRef,
        target.provider,
        target.model,
        target.credentialSlot,
      ]
    )
    return {
      invocationId,
      recipeNamespace,
      recipeName,
      hostRef: 'mcp-host/integration',
      attemptGeneration: 1,
      providerAttemptId,
      providerAttemptIndex: 1,
      status,
      target,
      reason: 'integration_provider_unavailable',
      ...(status === 'complete'
        ? {
            usage: {
              llmSecretName: 'openai-api-key',
              callerRef: 'integration-test',
              fallbackUsed: false,
              attemptCount: 1,
              inputTokens: 23,
              outputTokens: 11,
            },
          }
        : {}),
    }
  }

  /** Link a Codex ledger row to an SDK attempt, optionally already usage-ready. */
  async function linkCodex(
    client: PoolClient,
    input: PromptBridgeFinalizationInput,
    options: { ready?: boolean; providerAttemptId?: string; attemptIndex?: number } = {}
  ): Promise<string> {
    const attempt = await insertLlmProviderAttempt(client, {
      callerKind: 'recipe',
      hostRef: input.hostRef,
      recipeNamespace: input.recipeNamespace,
      recipeName: input.recipeName,
      invocationId: input.invocationId,
      attemptGeneration: input.attemptGeneration,
      providerAttemptIndex: options.attemptIndex ?? input.providerAttemptIndex,
      model: CODEX_TARGET.model,
      requestHash: 'a'.repeat(64),
      policyRevision: 1,
      policyHash: 'b'.repeat(64),
      budgetReservationId: randomUUID(),
      connectionRevision: 1,
      pluginWorkloadSdkProviderAttemptId: options.providerAttemptId ?? input.providerAttemptId,
    })
    if (options.ready) await markCodexUsageReady(client, attempt.id)
    return attempt.id
  }

  /**
   * The state the Codex proxy finalize leaves behind on success. Written
   * directly because driving the whole ticket lifecycle is the dual-ledger
   * suite's job, not this one's.
   */
  async function markCodexUsageReady(
    client: PoolClient,
    codexAttemptId: string,
    tokens: [number, number] = [12, 7]
  ): Promise<void> {
    await client.query(
      `UPDATE llm_provider_attempts
          SET status = 'finalized', outcome = 'success',
              usage_input_tokens = $2, usage_output_tokens = $3, finalized_at = now()
        WHERE id = $1`,
      [codexAttemptId, tokens[0], tokens[1]]
    )
  }

  /** Insert an extra physical attempt, as a failover would have left behind. */
  async function seedPriorAttempt(
    client: PoolClient,
    input: PromptBridgeFinalizationInput,
    options: { attemptIndex: number; status: string; codex?: boolean }
  ): Promise<string> {
    const id = randomUUID()
    const target = options.codex ? CODEX_TARGET : OPENAI_TARGET
    await client.query(
      `INSERT INTO plugin_workload_sdk_provider_attempts
         (id, invocation_id, recipe_namespace, recipe_name, attempt_generation,
          attempt_index, target_ref, provider, model, credential_slot, status,
          completed_at)
       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())`,
      [
        id,
        input.invocationId,
        input.recipeNamespace,
        input.recipeName,
        input.attemptGeneration,
        options.attemptIndex,
        target.targetRef,
        target.provider,
        target.model,
        target.credentialSlot,
        options.status,
      ]
    )
    return id
  }

  async function spendRows(providerAttemptId: string) {
    const result = await dbPool.query(
      `SELECT outcome, reason, input_tokens, output_tokens, usage_request_id, host_ref
         FROM plugin_workload_sdk_spend_outcomes
        WHERE provider_attempt_id = $1`,
      [providerAttemptId]
    )
    return result.rows as Array<Record<string, unknown>>
  }

  it('writes one immutable unknown outcome and makes an identical replay idempotent', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client)
      const first = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)

      expect(first).toMatchObject({
        status: 'provider_unavailable',
        outcome: 'unknown',
        idempotent: false,
      })

      const replayClient = await dbPool.connect()
      try {
        await begin(replayClient)
        const replay = await finalizePromptBridgeInTransaction(input, replayClient)
        await commit(replayClient)
        expect(replay).toMatchObject({
          status: 'provider_unavailable',
          outcome: 'unknown',
          idempotent: true,
        })
      } finally {
        replayClient.release()
      }

      const count = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE provider_attempt_id = $1`,
        [input.providerAttemptId]
      )
      expect(count.rows[0]?.count).toBe(1)
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('deterministically re-reads the sweeper outcome after ON CONFLICT', async () => {
    const seedClient = await dbPool.connect()
    const sweeperClient = await dbPool.connect()
    const finalizerClient = await dbPool.connect()
    try {
      await begin(seedClient)
      const input = await seedAttempt(seedClient, {
        status: 'provider_unavailable',
        leaseExpired: true,
      })
      await commit(seedClient)

      await begin(sweeperClient)
      // The sweeper writes its immutable outcome but keeps the transaction
      // open. The finalizer therefore cannot see the row in its first
      // READ COMMITTED ledger lookup, while the invocation row remains locked.
      await failStaleInvocationsInTransaction(1, sweeperClient)

      const initialLedgerRead = deferred<void>()
      const releaseInitialLedgerRead = deferred<void>()
      let ledgerReadCount = 0
      let conflictInsertRowCount: number | null = null
      const finalizerDb = {
        query: async (statement: string, values?: unknown[]) => {
          const result = await finalizerClient.query(statement, values)
          const isLedgerRead =
            /^\s*SELECT/i.test(statement) &&
            /FROM plugin_workload_sdk_spend_outcomes/i.test(statement)
          if (isLedgerRead) {
            ledgerReadCount += 1
            if (ledgerReadCount === 1 && result.rows.length === 0) {
              initialLedgerRead.resolve()
              await releaseInitialLedgerRead.promise
            }
          }
          if (
            /INSERT INTO plugin_workload_sdk_spend_outcomes/i.test(statement) &&
            /ON CONFLICT \(provider_attempt_id\) DO NOTHING/i.test(statement)
          ) {
            conflictInsertRowCount = result.rowCount
          }
          return result
        },
      }

      await begin(finalizerClient)
      const finalizationPromise = finalizePromptBridgeInTransaction(input, finalizerDb)

      // This barrier proves the finalizer completed its initial empty read
      // before the sweeper commits. Releasing it earlier would let the first
      // read observe the committed outcome and bypass the ON CONFLICT branch.
      await initialLedgerRead.promise
      await commit(sweeperClient)
      releaseInitialLedgerRead.resolve()

      const result = await finalizationPromise
      await commit(finalizerClient)

      expect(result).toMatchObject({
        status: 'provider_unavailable',
        outcome: 'unknown',
        idempotent: true,
        usageAccepted: false,
      })
      expect(ledgerReadCount).toBe(2)
      expect(conflictInsertRowCount).toBe(0)

      const count = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE provider_attempt_id = $1`,
        [input.providerAttemptId]
      )
      expect(count.rows[0]?.count).toBe(1)
    } finally {
      await Promise.all([
        sweeperClient.query('ROLLBACK').catch(() => undefined),
        finalizerClient.query('ROLLBACK').catch(() => undefined),
      ])
      seedClient.release()
      sweeperClient.release()
      finalizerClient.release()
    }
  })

  it('records exact usage and projects it once; an identical replay does not bill twice', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { status: 'complete' })
      const first = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)

      expect(first).toMatchObject({
        status: 'complete',
        outcome: 'exact',
        idempotent: false,
        usageAccepted: true,
      })

      const replayClient = await dbPool.connect()
      try {
        await begin(replayClient)
        const replay = await finalizePromptBridgeInTransaction(input, replayClient)
        await commit(replayClient)
        expect(replay).toMatchObject({
          status: 'complete',
          outcome: 'exact',
          idempotent: true,
          usageAccepted: true,
        })
      } finally {
        replayClient.release()
      }

      const counts = await dbPool.query(
        `SELECT
            (SELECT count(*) FROM plugin_workload_sdk_spend_outcomes WHERE provider_attempt_id = $1::uuid)::int AS spend_count,
            (SELECT count(*) FROM usage_events WHERE request_id = $1::uuid)::int AS usage_count,
            (SELECT count(*) FROM agent_run_events WHERE source_event_id = $1::text)::int AS trace_count,
            (SELECT count(*) FROM governed_event_stream stream
               JOIN agent_run_events event ON event.event_id = stream.event_id
              WHERE event.source_event_id = $1::text)::int AS stream_count`,
        [input.providerAttemptId]
      )
      expect(counts.rows[0]).toEqual({
        spend_count: 1,
        usage_count: 1,
        trace_count: 1,
        stream_count: 1,
      })
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('records a proven pre-provider failure as not_executed and keeps it idempotent', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { status: 'failed' })
      input.reason = 'integration_credential_resolution_failed'
      const first = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)
      expect(first).toMatchObject({ status: 'failed', outcome: 'not_executed', idempotent: false })

      const state = await dbPool.query(
        `SELECT inv.status AS invocation_status,
                receipt.status AS receipt_status,
                attempt.status AS provider_status,
                outcome.outcome
           FROM plugin_workload_sdk_invocations inv
           JOIN plugin_workload_sdk_invocation_attempts receipt ON receipt.invocation_id = inv.id
           JOIN plugin_workload_sdk_provider_attempts attempt ON attempt.id = $1
           JOIN plugin_workload_sdk_spend_outcomes outcome ON outcome.provider_attempt_id = $1
          WHERE inv.id = $2`,
        [input.providerAttemptId, input.invocationId]
      )
      expect(state.rows[0]).toEqual({
        invocation_status: 'failed',
        receipt_status: 'failed',
        provider_status: 'failed',
        outcome: 'not_executed',
      })
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('contains stale-sweeper/finalizer interleaving without a raw 500 or duplicate spend row', async () => {
    const seedClient = await dbPool.connect()
    const finalizerClient = await dbPool.connect()
    const sweeperClient = await dbPool.connect()
    try {
      await begin(seedClient)
      const input = await seedAttempt(seedClient, {
        status: 'provider_unavailable',
        leaseExpired: true,
      })
      await commit(seedClient)
      await Promise.all([begin(finalizerClient), begin(sweeperClient)])

      // Both operations contend on the invocation row lock. Do not await both
      // before committing: the loser cannot finish until the winner releases
      // its lock, so Promise.allSettled followed by two commits deadlocks the
      // test itself. Commit/rollback the first completed transaction, then
      // await the contender and release its transaction as well.
      const finalizerPromise = finalizePromptBridgeInTransaction(input, finalizerClient)
        .then(result => ({ kind: 'finalizer' as const, status: 'fulfilled' as const, result }))
        .catch(reason => ({ kind: 'finalizer' as const, status: 'rejected' as const, reason }))
      const sweeperPromise = failStaleInvocationsInTransaction(1, sweeperClient)
        .then(result => ({ kind: 'sweeper' as const, status: 'fulfilled' as const, result }))
        .catch(reason => ({ kind: 'sweeper' as const, status: 'rejected' as const, reason }))

      const first = await Promise.race([finalizerPromise, sweeperPromise])
      const release = async (winner: typeof first, client: PoolClient): Promise<void> => {
        if (winner.status === 'fulfilled') await commit(client)
        else await client.query('ROLLBACK')
      }
      if (first.kind === 'finalizer') await release(first, finalizerClient)
      else await release(first, sweeperClient)

      const second = first.kind === 'finalizer' ? await sweeperPromise : await finalizerPromise
      if (second.kind === 'finalizer') await release(second, finalizerClient)
      else await release(second, sweeperClient)

      const results = [first, second]
      expect(results.some(result => result.status === 'fulfilled')).toBe(true)
      const sweeperResult = results.find(result => result.kind === 'sweeper')!
      expect(sweeperResult.status).toBe('fulfilled')
      if (sweeperResult.status === 'rejected') throw sweeperResult.reason
      for (const result of results) {
        if (result.status === 'rejected' && result.kind === 'finalizer') {
          expect(result.reason).toBeInstanceOf(PromptBridgeFinalizationError)
          expect((result.reason as PromptBridgeFinalizationError).httpStatus).toBe(409)
        }
      }

      const count = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE provider_attempt_id = $1`,
        [input.providerAttemptId]
      )
      expect(count.rows[0]?.count).toBe(1)
    } finally {
      await Promise.all([
        finalizerClient.query('ROLLBACK').catch(() => undefined),
        sweeperClient.query('ROLLBACK').catch(() => undefined),
      ])
      seedClient.release()
      finalizerClient.release()
      sweeperClient.release()
    }
  })

  it('does not freeze oauth spend while a linked Codex attempt is still in flight', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { leaseExpired: true })
      await client.query(
        `UPDATE plugin_workload_sdk_provider_attempts
            SET provider = 'codex-subscription',
                model = 'gpt-5.1',
                credential_slot = '',
                target_ref = 'primary-codex'
          WHERE id = $1`,
        [input.providerAttemptId]
      )
      await insertLlmProviderAttempt(client, {
        callerKind: 'recipe',
        hostRef: input.hostRef,
        recipeNamespace: input.recipeNamespace,
        recipeName: input.recipeName,
        invocationId: input.invocationId,
        attemptGeneration: 1,
        providerAttemptIndex: 1,
        model: 'gpt-5.1',
        requestHash: 'a'.repeat(64),
        policyRevision: 1,
        policyHash: 'b'.repeat(64),
        budgetReservationId: randomUUID(),
        connectionRevision: 1,
        pluginWorkloadSdkProviderAttemptId: input.providerAttemptId,
      })
      const closed = await failStaleInvocationsInTransaction(1, client)
      await commit(client)

      expect(closed).toBe(0)
      const spend = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE provider_attempt_id = $1`,
        [input.providerAttemptId]
      )
      expect(spend.rows[0]?.count).toBe(0)
      const invocation = await dbPool.query(
        `SELECT status FROM plugin_workload_sdk_invocations WHERE id = $1`,
        [input.invocationId]
      )
      expect(invocation.rows[0]?.status).toBe('in_progress')
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('never rewrites the sweeper floor; late Codex usage is derived at read time', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { codex: true, leaseExpired: true })
      // Out of the in-flight grace, so the sweeper is free to close it.
      const codexAttemptId = await linkCodex(client, input)
      await client.query(
        `UPDATE llm_provider_attempts
            SET created_at = now() - make_interval(secs => $2::double precision / 1000)
          WHERE id = $1`,
        [codexAttemptId, CODEX_IN_FLIGHT_USAGE_GRACE_MS + 60_000]
      )
      expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)
      await commit(client)

      expect(await spendRows(input.providerAttemptId)).toEqual([
        {
          outcome: 'unknown',
          reason: 'stale_lease',
          input_tokens: null,
          output_tokens: null,
          usage_request_id: null,
          host_ref: null,
        },
      ])

      // Codex reports its usage after the sweep.
      const usageClient = await dbPool.connect()
      try {
        await begin(usageClient)
        await markCodexUsageReady(usageClient, codexAttemptId)
        await commit(usageClient)
      } finally {
        usageClient.release()
      }

      const replayClient = await dbPool.connect()
      try {
        await begin(replayClient)
        const replay = await finalizePromptBridgeInTransaction(
          {
            ...input,
            status: 'complete',
            usage: {
              llmSecretName: '',
              callerRef: 'integration-test',
              fallbackUsed: false,
              attemptCount: 1,
              inputTokens: 99,
              outputTokens: 99,
            },
          },
          replayClient
        )
        await commit(replayClient)
        // The outcome reports the derived truth...
        expect(replay).toMatchObject({ outcome: 'exact', idempotent: true, usageAccepted: false })
      } finally {
        replayClient.release()
      }

      // ...while the floor itself is byte-for-byte what the sweeper wrote.
      // An UPDATE here would have been a 42501 under the runtime role.
      expect(await spendRows(input.providerAttemptId)).toEqual([
        {
          outcome: 'unknown',
          reason: 'stale_lease',
          input_tokens: null,
          output_tokens: null,
          usage_request_id: null,
          host_ref: null,
        },
      ])
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('lets the sweeper freeze exact when the linked Codex usage already landed', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { codex: true, leaseExpired: true })
      await linkCodex(client, input, { ready: true })
      expect(await failStaleInvocationsInTransaction(1, client)).toBe(1)
      await commit(client)

      // Addendum A.4: a ready Codex row is the best floor provable without a
      // JWT, so the sweeper no longer under-reports it as unknown.
      expect(await spendRows(input.providerAttemptId)).toEqual([
        {
          outcome: 'exact',
          reason: 'stale_lease',
          input_tokens: 12,
          output_tokens: 7,
          usage_request_id: null,
          host_ref: null,
        },
      ])
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('closes a failed oauth attempt whose Codex row is ready as exact and non-revivable', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { codex: true, status: 'failed' })
      input.reason = 'provider_stream_failed'
      await linkCodex(client, input, { ready: true })
      // N-BLK-2(a): this used to persist `unknown` WITH tokens and die on
      // token_pair_check (23514 -> 500).
      const result = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)

      expect(result).toMatchObject({ status: 'failed', outcome: 'exact', idempotent: false })
      expect(await spendRows(input.providerAttemptId)).toEqual([
        {
          outcome: 'exact',
          reason: 'provider_stream_failed',
          input_tokens: 12,
          output_tokens: 7,
          usage_request_id: null,
          host_ref: input.hostRef,
        },
      ])

      // Guard 1.a: reviveFailedInvocation only matches `status = 'failed'`, so
      // persisting provider_unavailable is what stops the same idempotency key
      // from launching a second billable Codex call.
      const state = await dbPool.query(
        `SELECT inv.status AS invocation_status,
                receipt.status AS receipt_status,
                attempt.status AS provider_status
           FROM plugin_workload_sdk_invocations inv
           JOIN plugin_workload_sdk_invocation_attempts receipt
             ON receipt.invocation_id = inv.id
           JOIN plugin_workload_sdk_provider_attempts attempt ON attempt.id = $1
          WHERE inv.id = $2`,
        [input.providerAttemptId, input.invocationId]
      )
      expect(state.rows[0]).toEqual({
        invocation_status: 'provider_unavailable',
        receipt_status: 'provider_unavailable',
        provider_status: 'provider_unavailable',
      })
      const revivable = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_invocations
          WHERE id = $1 AND status = 'failed'`,
        [input.invocationId]
      )
      expect(revivable.rows[0]?.count).toBe(0)
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('resolves three identical failed calls while Codex usage lands between them', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { codex: true, status: 'failed' })
      const codexAttemptId = await linkCodex(client, input)
      await client.query(
        `UPDATE llm_provider_attempts SET status = 'finalized', outcome = 'unknown' WHERE id = $1`,
        [codexAttemptId]
      )
      const first = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)
      expect(first).toMatchObject({ outcome: 'unknown', idempotent: false })

      const usageClient = await dbPool.connect()
      try {
        await begin(usageClient)
        await markCodexUsageReady(usageClient, codexAttemptId)
        await commit(usageClient)
      } finally {
        usageClient.release()
      }

      // Before the fix the second call rewrote the row to `exact` and the third
      // then 409'd against its own predecessor. The floor is immutable now, so
      // both replays agree.
      for (const _attempt of [2, 3]) {
        const replayClient = await dbPool.connect()
        try {
          await begin(replayClient)
          const replay = await finalizePromptBridgeInTransaction(input, replayClient)
          await commit(replayClient)
          expect(replay).toMatchObject({
            status: 'failed',
            outcome: 'exact',
            idempotent: true,
            usageAccepted: false,
          })
        } finally {
          replayClient.release()
        }
      }
      expect(await spendRows(input.providerAttemptId)).toEqual([
        {
          outcome: 'unknown',
          reason: 'integration_provider_unavailable',
          input_tokens: null,
          output_tokens: null,
          usage_request_id: null,
          host_ref: input.hostRef,
        },
      ])
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('settles the attempts a successful failover displaced', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { codex: true, status: 'complete' })
      // The winner is attempt 3; 1 and 2 are the terminal attempts a
      // successful failover leaves behind and never finalizes.
      await client.query(
        `UPDATE plugin_workload_sdk_provider_attempts SET attempt_index = 3 WHERE id = $1`,
        [input.providerAttemptId]
      )
      input.providerAttemptIndex = 3
      const failedId = await seedPriorAttempt(client, input, { attemptIndex: 1, status: 'failed' })
      const unavailableId = await seedPriorAttempt(client, input, {
        attemptIndex: 2,
        status: 'provider_unavailable',
        codex: true,
      })
      await linkCodex(client, input, {
        ready: true,
        providerAttemptId: unavailableId,
        attemptIndex: 2,
      })
      await linkCodex(client, input, { ready: true, attemptIndex: 3 })

      const result = await finalizePromptBridgeInTransaction(input, client)
      await commit(client)
      expect(result).toMatchObject({ outcome: 'exact', idempotent: false })

      expect(await spendRows(failedId)).toEqual([
        {
          outcome: 'not_executed',
          reason: 'prior_attempt_failed',
          input_tokens: null,
          output_tokens: null,
          usage_request_id: null,
          host_ref: input.hostRef,
        },
      ])
      // A displaced oauth attempt whose Codex row is ready still owes exact
      // spend, even though the host never reported it.
      expect(await spendRows(unavailableId)).toEqual([
        {
          outcome: 'exact',
          reason: 'prior_attempt_provider_unavailable',
          input_tokens: 12,
          output_tokens: 7,
          usage_request_id: null,
          host_ref: input.hostRef,
        },
      ])

      // A replay is pure reads: no second row, no rewrite.
      const replayClient = await dbPool.connect()
      try {
        await begin(replayClient)
        await expect(finalizePromptBridgeInTransaction(input, replayClient)).resolves.toMatchObject(
          { idempotent: true, outcome: 'exact' }
        )
        await commit(replayClient)
      } finally {
        replayClient.release()
      }
      const total = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE invocation_id = $1`,
        [input.invocationId]
      )
      expect(total.rows[0]?.count).toBe(3)
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })

  it('fails loudly when a displaced attempt is not terminal', async () => {
    const client = await dbPool.connect()
    try {
      await begin(client)
      const input = await seedAttempt(client, { status: 'complete' })
      await client.query(
        `UPDATE plugin_workload_sdk_provider_attempts SET attempt_index = 2 WHERE id = $1`,
        [input.providerAttemptId]
      )
      input.providerAttemptIndex = 2
      await seedPriorAttempt(client, input, { attemptIndex: 1, status: 'in_progress' })

      await expect(finalizePromptBridgeInTransaction(input, client)).rejects.toThrow(
        /is not terminal \(status=in_progress\); the reservation fence is broken/
      )
      await client.query('ROLLBACK')

      // The whole transaction rolled back, so no half-settled ledger survives.
      const total = await dbPool.query(
        `SELECT count(*)::int AS count
           FROM plugin_workload_sdk_spend_outcomes
          WHERE invocation_id = $1`,
        [input.invocationId]
      )
      expect(total.rows[0]?.count).toBe(0)
    } finally {
      await client.query('ROLLBACK').catch(() => undefined)
      client.release()
    }
  })
})