Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
20.0 KB
Raw Blame
/**
 * #753: a Codex refresh token the vendor rejects with `invalid_grant` marks the
 * connection `reauth_required`, the way the Grok broker already does.
 *
 * Everything below the OpenAI network boundary is real: the refresh lock, the
 * revision fence, the status write, the catalog sync result and the cron tick.
 * Only `fetchFn` (the token endpoint), the catalog transport and the ConfigMap
 * writer are doubled, because they are systems outside this process.
 */
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { createHash, randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import { initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import type { CodexCatalogTransport } from '../src/services/codexSubscriptionCatalog.js'
import {
  acquireCodexSubscriptionRefreshLock,
  getSafeCodexSubscriptionConnection,
  insertInitialCodexSubscriptionConnection,
  listLiveCodexSubscriptionConnections,
  loadCodexSubscriptionSecrets,
  updateCodexAccessTokenInPlace,
} from '../src/services/codexSubscriptionConnection.js'
import {
  CODEX_OAUTH_TOKEN_URL,
  type CodexOAuthDeps,
  CodexSubscriptionOAuthError,
  ensureFreshCodexAccessToken,
  refreshCodexSubscriptionConnection,
  runCodexCatalogSync,
} from '../src/services/codexSubscriptionOAuth.js'
import {
  type AllowedModelsConfigMapMaterializer,
  syncOutcomeChangedTheRow,
} from '../src/services/llmAllowedModelsConfigMap.js'
import {
  type SubscriptionBrokerPort,
  reconcileSubscriptionCatalogs,
} from '../src/services/subscriptionCatalogSyncCron.js'
import './realPostgres.requirement.ts'

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

function jsonResponse(status: number, body: Record<string, unknown>): Response {
  return { ok: status >= 200 && status < 300, status, json: async () => body } as Response
}

/**
 * The token endpoint answers every refresh with `status` / `body`. `beforeReply`
 * runs while the refresh lock is held, which is where a concurrent writer can
 * move the revision or the lock under the exchange.
 */
function tokenEndpoint(
  status: number,
  body: Record<string, unknown>,
  beforeReply: () => Promise<void> = async () => {}
) {
  return vi.fn(async (url: string | URL) => {
    if (String(url) !== CODEX_OAUTH_TOKEN_URL) return jsonResponse(404, {})
    await beforeReply()
    return jsonResponse(status, body)
  })
}

function readyTransport(): { transport: CodexCatalogTransport; calls: number[] } {
  const calls: number[] = []
  return {
    calls,
    transport: {
      listModels: async () => {
        calls.push(1)
        return { outcome: 'ready', models: [{ model: 'gpt-5' }] }
      },
    },
  }
}

describeRealPostgres('Codex refresh rejected by the vendor on real PostgreSQL (#753)', () => {
  const database = `codex_refresh_rejected_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? `postgresql://postgres@${['127', '0', '0', '1'].join('.')}/postgres`,
    database
  )
  let adminPool: Pool
  let pool: Pool

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    pool = new Pool({ connectionString })
    await initDb({ connect: () => pool.connect() })
  }, 60_000)

  afterAll(async () => {
    try {
      await pool?.end()
      if (adminPool) {
        await adminPool
          .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
          .catch(() => undefined)
      }
    } finally {
      await adminPool?.end()
    }
  })

  /**
   * A connected grant with no access token, so every ensure-fresh call goes to
   * the token endpoint instead of returning early.
   */
  async function seedConnected(connectionKey: string): Promise<void> {
    await insertInitialCodexSubscriptionConnection(
      pool,
      KEY,
      {
        refreshToken: `refresh-${connectionKey}`,
        accountFingerprint: createHash('sha256').update(connectionKey, 'utf8').digest('hex'),
      },
      connectionKey
    )
  }

  function deps(connectionKey: string, fetchFn: typeof fetch): CodexOAuthDeps {
    return {
      db: pool,
      encryptionKey: KEY,
      fetchFn,
      clientId: 'app_test_client',
      redirectUri: 'https://control.example/api/v1/auth/codex-subscription/callback',
      enabled: true,
      connectionKey,
    }
  }

  function codexPort(
    transport: CodexCatalogTransport,
    fetchFn: typeof fetch,
    syncedKeys: string[]
  ): SubscriptionBrokerPort {
    return {
      broker: 'codex-subscription',
      enabled: true,
      listConnections: () => listLiveCodexSubscriptionConnections(pool),
      syncCatalog: key => {
        syncedKeys.push(key)
        return runCodexCatalogSync(deps(key, fetchFn), key, transport)
      },
    }
  }

  async function statusOf(connectionKey: string): Promise<string | undefined> {
    return (await getSafeCodexSubscriptionConnection(pool, connectionKey))?.status
  }

  it('T-753a a 400 invalid_grant on ensure-fresh marks the row reauth_required and says so', async () => {
    const key = 'codex-753a'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toBeInstanceOf(CodexSubscriptionOAuthError)
    expect(failure).toMatchObject({ code: 'reauth_required', persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('reauth_required')
  })

  it('T-753a a 401 invalid_grant on the manual refresh marks the row the same way', async () => {
    const key = 'codex-753a-manual'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(401, { error: { code: 'invalid_grant' } })

    const failure = await refreshCodexSubscriptionConnection(
      deps(key, fetchFn as typeof fetch)
    ).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'reauth_required', persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('reauth_required')
  })

  it('T-753b runCodexCatalogSync reports the written row as persisted, so a republish is owed', async () => {
    const key = 'codex-753b'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' })
    const { transport, calls } = readyTransport()

    const synced = await runCodexCatalogSync(deps(key, fetchFn as typeof fetch), key, transport)

    expect(synced).toMatchObject({
      ok: false,
      catalogStatus: 'never_synced',
      reason: 'reauth_required',
      persisted: true,
    })
    expect(syncOutcomeChangedTheRow(synced)).toBe(true)
    // Liveness witness for the untouched catalog: the refresh really ran.
    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(calls).toEqual([])
  })

  it('T-753c the cron marks the rejected grant once and skips it on the next tick', async () => {
    const key = 'codex-753c'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' })
    const { transport } = readyTransport()
    const published: number[] = []
    const materializer: AllowedModelsConfigMapMaterializer = {
      materialize: async () => {
        published.push(Date.now())
      },
    }

    // Earlier cases in this file leave rows behind, so the expected counts are
    // read from the database rather than assumed.
    const live = await listLiveCodexSubscriptionConnections(pool)
    const connectedBefore = live
      .filter(row => row.status === 'connected')
      .map(row => row.connectionKey)
    expect(connectedBefore).toContain(key)

    const firstKeys: string[] = []
    const first = await reconcileSubscriptionCatalogs({
      brokers: [codexPort(transport, fetchFn as typeof fetch, firstKeys)],
      materializer,
    })
    const callsAfterFirstTick = fetchFn.mock.calls.length
    const publishesAfterFirstTick = published.length

    const secondKeys: string[] = []
    const second = await reconcileSubscriptionCatalogs({
      brokers: [codexPort(transport, fetchFn as typeof fetch, secondKeys)],
      materializer,
    })

    // Tick 1: every connected grant reached the token endpoint once, and each
    // rejection is a recorded change rather than a failure.
    expect(firstKeys).toEqual(connectedBefore)
    expect(callsAfterFirstTick).toBe(connectedBefore.length)
    expect(first).toMatchObject({
      synced: 0,
      degraded: connectedBefore.length,
      failed: 0,
      skipped: live.length - connectedBefore.length,
    })
    expect(publishesAfterFirstTick).toBe(1)
    expect(await statusOf(key)).toBe('reauth_required')
    // The cron publishes on every tick by design (subscriptionCatalogSyncCron.ts),
    // so tick 2 adds exactly one more.
    expect(published).toHaveLength(2)
    // Tick 2 listed every grant (all of them count as skipped) and sent no dead
    // refresh token again.
    expect(second).toMatchObject({ synced: 0, degraded: 0, failed: 0, skipped: live.length })
    expect(secondKeys).toEqual([])
    expect(fetchFn.mock.calls.length).toBe(callsAfterFirstTick)
  })

  it('T-753d invalid_grant after the revision moved is a lost race and leaves the row connected', async () => {
    const key = 'codex-753d-revision'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' }, async () => {
      await pool.query(
        `UPDATE codex_subscription_connections
            SET credential_revision = credential_revision + 1
          WHERE connection_key = $1`,
        [key]
      )
    })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    // `stale_revision` only comes out of the invalid_grant branch here, so it
    // is also the witness that the branch read the fence.
    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'stale_revision' })
    expect(await statusOf(key)).toBe('connected')
  })

  it('T-753d invalid_grant after the refresh lock was lost is a lost race too', async () => {
    const key = 'codex-753d-lock'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' }, async () => {
      await pool.query(
        `UPDATE codex_subscription_connections
            SET refresh_lock_token = NULL,
                refresh_lock_expires_at = NULL
          WHERE connection_key = $1`,
        [key]
      )
    })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'stale_revision' })
    expect(await statusOf(key)).toBe('connected')
  })

  it('T-753j invalid_grant after another holder took the lock and refreshed in place is a lost race', async () => {
    const key = 'codex-753j-takeover'
    await seedConnected(key)
    const takeover: { locked?: boolean; revision?: number } = {}
    // Our lock expires mid-exchange; another holder acquires it with its own
    // token and rotates the refresh token in place. In-place refresh does not
    // bump credential_revision, so only the lock token tells the two apart.
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' }, async () => {
      await pool.query(
        `UPDATE codex_subscription_connections
            SET refresh_lock_expires_at = now() - interval '1 second'
          WHERE connection_key = $1`,
        [key]
      )
      const locked = await acquireCodexSubscriptionRefreshLock(pool, 'other-holder', 30_000, key)
      const before = await loadCodexSubscriptionSecrets(pool, KEY, key)
      if (!before) throw new Error('seeded grant disappeared before the takeover')
      await updateCodexAccessTokenInPlace(
        pool,
        KEY,
        before.credentialRevision,
        { accessToken: 'access-other-holder', refreshToken: 'refresh-other-holder' },
        key
      )
      takeover.locked = locked
      takeover.revision = before.credentialRevision
    })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )
    const after = await loadCodexSubscriptionSecrets(pool, KEY, key)

    // Liveness witness: the exchange ran, the other holder really took the
    // lock, and its write is the one stored, at the revision we also read.
    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(takeover.locked).toBe(true)
    expect(typeof takeover.revision).toBe('number')
    expect(after).toMatchObject({
      refreshToken: 'refresh-other-holder',
      accessToken: 'access-other-holder',
      credentialRevision: takeover.revision,
    })
    expect(failure).toBeInstanceOf(CodexSubscriptionOAuthError)
    expect(failure).toMatchObject({ code: 'stale_revision', persistedConnectionStatus: false })
    expect(await statusOf(key)).toBe('connected')
  })

  it('T-753h a revoke landing between the fence check and the mark is a lost race, not a persisted reauth', async () => {
    const key = 'codex-753h-revoked'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_grant' })
    // The fence reads pass; the revoke lands just before the fenced UPDATE, so
    // the mark matches no row. Only the status write is intercepted; every other
    // statement runs unchanged against the real database.
    let markAttempts = 0
    const racingDb = {
      query: async (text: string, values?: unknown[]) => {
        if (text.includes("SET status = 'reauth_required'")) {
          markAttempts += 1
          await pool.query(
            `UPDATE codex_subscription_connections
                SET revoked_at = now()
              WHERE connection_key = $1`,
            [key]
          )
        }
        return pool.query(text, values)
      },
    }

    const failure = await ensureFreshCodexAccessToken({
      ...deps(key, fetchFn as typeof fetch),
      db: racingDb,
    }).then(
      () => null,
      (err: unknown) => err
    )

    // Liveness witness: the exchange ran and the mark was attempted, so the
    // outcome below is the empty UPDATE and not an earlier exit.
    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(markAttempts).toBe(1)
    expect(failure).toBeInstanceOf(CodexSubscriptionOAuthError)
    expect(failure).toMatchObject({ code: 'stale_revision', persistedConnectionStatus: false })
    // Read the table directly: the safe reader filters revoked rows, so it
    // returns undefined here whatever status the mark left behind.
    const { rows } = await pool.query<{ status: string; revoked_at: Date | null }>(
      `SELECT status, revoked_at FROM codex_subscription_connections WHERE connection_key = $1`,
      [key]
    )
    expect(rows).toHaveLength(1)
    // Witness: the revoke really landed, so the row is the one the race produced.
    expect(rows[0]?.revoked_at).toBeInstanceOf(Date)
    expect(rows[0]?.status).not.toBe('reauth_required')
  })

  it('T-753e a 503 from the token endpoint stays provider_unavailable and leaves the row connected', async () => {
    const key = 'codex-753e-503'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(503, { error: 'invalid_grant' })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'provider_unavailable' })
    expect(failure).not.toMatchObject({ persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('connected')
  })

  it('T-753e a 400 with another error code stays provider_unavailable', async () => {
    const key = 'codex-753e-other'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, { error: 'invalid_request' })

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'provider_unavailable' })
    expect(await statusOf(key)).toBe('connected')
  })

  /**
   * The official Codex client treats a refresh as permanent on any 401, on the
   * three refresh-token codes at any status (case-insensitive), and on a 400
   * `invalid_grant` (`classify_refresh_token_failure` in openai/codex
   * `codex-rs/login/src/auth/manager.rs`). The body is the shape the token
   * endpoint returns for these codes.
   */
  function upstreamRefreshError(code: string): Record<string, unknown> {
    return {
      error: {
        message: 'Your refresh token could not be used to obtain a new access token.',
        type: 'invalid_request_error',
        code,
      },
    }
  }

  it.each([
    [401, 'refresh_token_expired', 'codex-753i-401-expired'],
    [401, 'refresh_token_reused', 'codex-753i-401-reused'],
    [401, 'refresh_token_invalidated', 'codex-753i-401-invalidated'],
    [400, 'refresh_token_expired', 'codex-753i-400-expired'],
    [400, 'refresh_token_reused', 'codex-753i-400-reused'],
    [400, 'refresh_token_invalidated', 'codex-753i-400-invalidated'],
    [400, 'REFRESH_TOKEN_REUSED', 'codex-753i-400-reused-upper'],
  ])('T-753i a %i with code %s marks the row reauth_required', async (status, code, key) => {
    await seedConnected(key)
    const fetchFn = tokenEndpoint(status, upstreamRefreshError(code))

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toBeInstanceOf(CodexSubscriptionOAuthError)
    expect(failure).toMatchObject({ code: 'reauth_required', persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('reauth_required')
  })

  it('T-753i a bare 401 with no error code marks the row reauth_required', async () => {
    const key = 'codex-753i-bare-401'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(401, {})

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'reauth_required', persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('reauth_required')
  })

  it('T-753i a 400 INVALID_GRANT in another case is the same permanent rejection', async () => {
    const key = 'codex-753i-400-invalid-grant-upper'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, upstreamRefreshError('INVALID_GRANT'))

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'reauth_required', persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('reauth_required')
  })

  it('T-753i control: a 400 with an unrelated code in the upstream shape stays provider_unavailable', async () => {
    const key = 'codex-753i-400-unrelated'
    await seedConnected(key)
    const fetchFn = tokenEndpoint(400, upstreamRefreshError('unsupported_grant_type'))

    const failure = await ensureFreshCodexAccessToken(deps(key, fetchFn as typeof fetch)).then(
      () => null,
      (err: unknown) => err
    )

    // Liveness witness: the exchange reached the token endpoint once and the
    // row is read back as present, so "connected" is not an absent row.
    expect(fetchFn).toHaveBeenCalledTimes(1)
    expect(failure).toMatchObject({ code: 'provider_unavailable' })
    expect(failure).not.toMatchObject({ persistedConnectionStatus: true })
    expect(await statusOf(key)).toBe('connected')
  })
})