Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.8 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { config } from '../src/config.js'
import { llmAllowlistConfigMapWriteFailuresTotal } from '../src/observability/metrics.js'
import {
  GrokProviderAttemptRedeemError,
  redeemGrokProviderAttempt,
} from '../src/services/grokProviderAttemptRedemption.js'
import * as ticket from '../src/services/grokProviderAttemptTicket.js'
import * as connection from '../src/services/grokSubscriptionConnection.js'
import { GrokSubscriptionOAuthError } from '../src/services/grokSubscriptionOAuth.js'
import * as store from '../src/services/llmProviderAttemptStore.js'

vi.mock('../src/db.js', () => ({
  pool: { query: vi.fn() },
  withTransaction: async (work: (tx: { query: typeof vi.fn }) => unknown) =>
    work({ query: vi.fn() }),
}))

const CONNECTION_ID = '11111111-1111-4111-8111-111111111111'
const CLAIMS = {
  jti: '22222222-2222-4222-8222-222222222222',
  typ: 'grok-execution-ticket' as const,
  sub: 'host/research-host',
  hostRef: 'research-host',
  invocationId: 'invocation-1',
  attemptGeneration: 1,
  providerAttemptId: '33333333-3333-4333-8333-333333333333',
  providerAttemptIndex: 1,
  provider: 'grok-subscription' as const,
  model: 'grok-4.6',
  requestHash: 'a'.repeat(64),
  policyRevision: 4,
  policyHash: 'b'.repeat(64),
  budgetReservationId: 'unbudgeted',
  connectionRevision: 3,
  connectionId: CONNECTION_ID,
}

describe('redeemGrokProviderAttempt', () => {
  beforeEach(() => {
    config.grokSubscriptionEnabled = true
    vi.spyOn(ticket, 'verifyGrokExecutionTicket').mockReturnValue(CLAIMS)
  })

  it('rejects a replayed ticket before rotating the refresh token', async () => {
    const ensureFresh = vi.fn()
    vi.spyOn(store, 'peekLlmProviderAttemptTicket').mockResolvedValue({
      jti: CLAIMS.jti,
      providerAttemptId: CLAIMS.providerAttemptId,
      status: 'redeemed',
      expiresAt: new Date(Date.now() + 30_000),
      receiptHash: null,
    })
    await expect(
      redeemGrokProviderAttempt(
        { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },
        {
          enabled: true,
          db: { query: vi.fn() },
          withTransaction: async work => work({ query: vi.fn() }),
          loadSecrets: vi.fn(),
          getConnectionById: connection.getSafeGrokSubscriptionConnectionById,
          encryptionKey: Buffer.alloc(32),
          ensureFreshAccessToken: ensureFresh,
        }
      )
    ).rejects.toMatchObject({
      code: 'ticket_replayed',
    } satisfies Partial<GrokProviderAttemptRedeemError>)
    expect(ensureFresh).not.toHaveBeenCalled()
  })

  it('rejects a Codex-shaped ticket before loading Grok secrets', async () => {
    vi.spyOn(ticket, 'verifyGrokExecutionTicket').mockReturnValue(null)
    const loadSecrets = vi.fn()
    await expect(
      redeemGrokProviderAttempt(
        { executionTicket: 'codex-ticket', requestHash: CLAIMS.requestHash },
        {
          enabled: true,
          db: { query: vi.fn() },
          withTransaction: async work => work({ query: vi.fn() }),
          loadSecrets,
          getConnectionById: connection.getSafeGrokSubscriptionConnectionById,
          encryptionKey: Buffer.alloc(32),
        }
      )
    ).rejects.toMatchObject({ code: 'ticket_invalid' })
    expect(loadSecrets).not.toHaveBeenCalled()
  })

  it('maps reauth_required refresh to a non-retryable no_grant', async () => {
    vi.spyOn(store, 'peekLlmProviderAttemptTicket').mockResolvedValue({
      jti: CLAIMS.jti,
      providerAttemptId: CLAIMS.providerAttemptId,
      status: 'issued',
      expiresAt: new Date(Date.now() + 30_000),
      receiptHash: null,
    })
    vi.spyOn(store, 'loadLlmProviderAttempt').mockResolvedValue({
      id: CLAIMS.providerAttemptId,
      provider: 'grok-subscription',
      connectionId: CONNECTION_ID,
      status: 'authorized',
    } as never)
    await expect(
      redeemGrokProviderAttempt(
        { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },
        {
          enabled: true,
          db: { query: vi.fn() },
          withTransaction: async work => work({ query: vi.fn() }),
          loadSecrets: vi.fn(),
          getConnectionById: vi.fn().mockResolvedValue({ connectionKey: 'team-grok' }),
          encryptionKey: Buffer.alloc(32),
          ensureFreshAccessToken: async () => {
            throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected')
          },
        }
      )
    ).rejects.toMatchObject({ code: 'no_grant' })
  })

  it('maps a missing connection row to connection_unavailable instead of an unmapped error', async () => {
    vi.spyOn(store, 'peekLlmProviderAttemptTicket').mockResolvedValue({
      jti: CLAIMS.jti,
      providerAttemptId: CLAIMS.providerAttemptId,
      status: 'issued',
      expiresAt: new Date(Date.now() + 30_000),
      receiptHash: null,
    })
    vi.spyOn(store, 'loadLlmProviderAttempt').mockResolvedValue({
      id: CLAIMS.providerAttemptId,
      provider: 'grok-subscription',
      connectionId: CONNECTION_ID,
      status: 'authorized',
    } as never)
    const ensureFresh = vi.fn(async (connectionKey?: string) => {
      connection.assertGrokConnectionKey(connectionKey ?? '')
    })
    const withTransaction = vi.fn()
    await expect(
      redeemGrokProviderAttempt(
        { executionTicket: 'ticket', requestHash: CLAIMS.requestHash },
        {
          enabled: true,
          db: { query: vi.fn() },
          withTransaction: withTransaction as never,
          loadSecrets: vi.fn(),
          getConnectionById: vi.fn().mockResolvedValue(null),
          encryptionKey: Buffer.alloc(32),
          ensureFreshAccessToken: ensureFresh,
        }
      )
    ).rejects.toMatchObject({
      name: 'GrokProviderAttemptRedeemError',
      code: 'connection_unavailable',
    })
    expect(ensureFresh).not.toHaveBeenCalled()
    expect(withTransaction).not.toHaveBeenCalled()
  })
})

describe('redeemGrokProviderAttempt error-path matrix', () => {
  const ISSUED_TICKET = {
    jti: CLAIMS.jti,
    providerAttemptId: CLAIMS.providerAttemptId,
    status: 'issued' as const,
    expiresAt: new Date(Date.now() + 30_000),
    receiptHash: null,
  }

  function attemptRow(overrides: Record<string, unknown> = {}) {
    return {
      id: CLAIMS.providerAttemptId,
      provider: 'grok-subscription',
      status: 'authorized',
      connectionId: CONNECTION_ID,
      requestHash: CLAIMS.requestHash,
      model: CLAIMS.model,
      hostRef: CLAIMS.hostRef,
      invocationId: CLAIMS.invocationId,
      attemptGeneration: CLAIMS.attemptGeneration,
      policyHash: CLAIMS.policyHash,
      policyRevision: CLAIMS.policyRevision,
      budgetReservationId: CLAIMS.budgetReservationId,
      connectionRevision: CLAIMS.connectionRevision,
      ...overrides,
    }
  }

  function liveConnection(overrides: Record<string, unknown> = {}) {
    return {
      id: CONNECTION_ID,
      connectionKey: 'team-grok',
      status: 'connected',
      revokedAt: null,
      credentialRevision: CLAIMS.connectionRevision,
      ...overrides,
    }
  }

  type Scenario = {
    enabled?: boolean
    claims?: Partial<typeof CLAIMS> | null
    input?: Partial<{ requestHash: string; model: string; hostRef: string }>
    peek?: unknown
    previewAttempt?: unknown
    assigned?: unknown
    ensureFresh?: ((key?: string) => Promise<void>) | null
    locked?: unknown
    txAttempt?: unknown
    connection?: unknown
    markRedeemed?: boolean
    secrets?: unknown
    publishAllowlist?: () => Promise<void>
  }

  function arrange(scenario: Scenario = {}) {
    vi.spyOn(ticket, 'verifyGrokExecutionTicket').mockReturnValue(
      scenario.claims === null ? null : ({ ...CLAIMS, ...scenario.claims } as typeof CLAIMS)
    )
    const peek = vi
      .spyOn(store, 'peekLlmProviderAttemptTicket')
      .mockResolvedValue(('peek' in scenario ? scenario.peek : ISSUED_TICKET) as never)
    const previewAttempt = 'previewAttempt' in scenario ? scenario.previewAttempt : attemptRow()
    const txAttempt = 'txAttempt' in scenario ? scenario.txAttempt : attemptRow()
    vi.spyOn(store, 'loadLlmProviderAttempt')
      .mockResolvedValueOnce(previewAttempt as never)
      .mockResolvedValueOnce(txAttempt as never)
    const lock = vi
      .spyOn(store, 'lockLlmProviderAttemptTicket')
      .mockResolvedValue(('locked' in scenario ? scenario.locked : ISSUED_TICKET) as never)
    const connectionById = vi
      .spyOn(connection, 'getSafeGrokSubscriptionConnectionById')
      .mockResolvedValue(
        ('connection' in scenario ? scenario.connection : liveConnection()) as never
      )
    const markRedeemed = vi
      .spyOn(store, 'markLlmProviderAttemptTicketRedeemed')
      .mockResolvedValue(scenario.markRedeemed ?? true)
    const loadSecrets = vi.fn().mockResolvedValue(
      'secrets' in scenario
        ? scenario.secrets
        : {
            refreshToken: 'refresh-secret',
            accessToken: 'access-secret',
            accessTokenExpiresAt: null,
            credentialRevision: CLAIMS.connectionRevision,
          }
    )
    const ensureFresh =
      scenario.ensureFresh === null ? undefined : vi.fn(scenario.ensureFresh ?? (async () => {}))
    const getConnectionById = vi
      .fn()
      .mockResolvedValue('assigned' in scenario ? scenario.assigned : liveConnection())
    const tx = { query: vi.fn() }
    const withTransaction = vi.fn(async (work: (db: typeof tx) => unknown) => work(tx))
    const publishAllowlist = vi.fn(scenario.publishAllowlist ?? (async () => {}))
    const run = () =>
      redeemGrokProviderAttempt(
        {
          executionTicket: 'ticket',
          requestHash: CLAIMS.requestHash,
          ...scenario.input,
        },
        {
          enabled: scenario.enabled ?? true,
          db: { query: vi.fn() },
          withTransaction: withTransaction as never,
          loadSecrets,
          getConnectionById,
          encryptionKey: Buffer.alloc(32),
          publishAllowlist,
          ...(ensureFresh ? { ensureFreshAccessToken: ensureFresh } : {}),
        }
      )
    return {
      run,
      publishAllowlist,
      peek,
      lock,
      connectionById,
      markRedeemed,
      loadSecrets,
      ensureFresh,
      getConnectionById,
      withTransaction,
    }
  }

  afterEach(() => {
    vi.restoreAllMocks()
  })

  it('redeems when every binding matches and never returns the refresh token', async () => {
    const h = arrange()
    const result = await h.run()
    expect(result.accessToken).toBe('access-secret')
    expect(result.expiryClass).toBe('upstream_managed')
    expect(result.transport.servedModel).toBe(CLAIMS.model)
    // The per-attempt total cap the proxy enforces (30 min).
    expect(result.transport.maxStreamDurationMs).toBe(1_800_000)
    expect(JSON.stringify(result)).not.toContain('refresh-secret')
    expect(h.ensureFresh).toHaveBeenCalledWith('team-grok')
    expect(h.markRedeemed).toHaveBeenCalledTimes(1)
    expect(h.loadSecrets).toHaveBeenCalledWith(expect.anything(), expect.any(Buffer), 'team-grok')
  })

  it('classifies an access token expiring within the hour as short_lived', async () => {
    const h = arrange({
      secrets: {
        refreshToken: 'refresh-secret',
        accessToken: 'access-secret',
        accessTokenExpiresAt: new Date(Date.now() + 60_000),
        credentialRevision: CLAIMS.connectionRevision,
      },
    })
    await expect(h.run()).resolves.toMatchObject({ expiryClass: 'short_lived' })
  })

  it('rejects while disabled before verifying the ticket', async () => {
    const h = arrange({ enabled: false })
    await expect(h.run()).rejects.toMatchObject({ code: 'disabled' })
    expect(ticket.verifyGrokExecutionTicket).not.toHaveBeenCalled()
    expect(h.peek).not.toHaveBeenCalled()
  })

  type PreTx = [string, Scenario, string]
  const preTransaction: PreTx[] = [
    ['an unverifiable ticket', { claims: null }, 'ticket_invalid'],
    [
      'a non-Grok ticket provider',
      { claims: { provider: 'codex-subscription' as never } },
      'ticket_invalid',
    ],
    [
      'a request hash mismatch',
      { input: { requestHash: 'c'.repeat(64) } },
      'request_hash_mismatch',
    ],
    ['a model mismatch', { input: { model: 'grok-other' } }, 'ticket_invalid'],
    ['a hostRef mismatch', { input: { hostRef: 'other-host' } }, 'ticket_invalid'],
    ['an unregistered ticket', { peek: null }, 'ticket_invalid'],
    [
      'an already-redeemed ticket',
      { peek: { ...ISSUED_TICKET, status: 'redeemed' } },
      'ticket_replayed',
    ],
    ['a missing attempt', { previewAttempt: null }, 'ticket_invalid'],
    [
      'a Codex attempt row',
      { previewAttempt: attemptRow({ provider: 'codex-subscription' }) },
      'ticket_invalid',
    ],
    [
      'an attempt bound to another connection',
      { previewAttempt: attemptRow({ connectionId: '44444444-4444-4444-8444-444444444444' }) },
      'ticket_invalid',
    ],
    ['a vanished connection row', { assigned: null }, 'connection_unavailable'],
  ]

  it.each(preTransaction)(
    'rejects %s before refreshing tokens or opening the transaction',
    async (_label, scenario, code) => {
      const h = arrange(scenario)
      await expect(h.run()).rejects.toMatchObject({ name: 'GrokProviderAttemptRedeemError', code })
      expect(h.ensureFresh).not.toHaveBeenCalled()
      expect(h.withTransaction).not.toHaveBeenCalled()
      expect(h.loadSecrets).not.toHaveBeenCalled()
    }
  )

  it.each([
    ['no_grant', 'no_grant'],
    ['not_connected', 'no_grant'],
    ['reauth_required', 'no_grant'],
    ['provider_unavailable', 'provider_unavailable'],
    ['refresh_in_flight', 'connection_unavailable'],
    ['stale_revision', 'connection_unavailable'],
  ] as const)('maps refresh failure %s to %s without redeeming', async (oauthCode, code) => {
    const h = arrange({
      ensureFresh: async () => {
        throw new GrokSubscriptionOAuthError(oauthCode, 'refresh failed')
      },
    })
    await expect(h.run()).rejects.toMatchObject({ code })
    expect(h.withTransaction).not.toHaveBeenCalled()
    expect(h.markRedeemed).not.toHaveBeenCalled()
  })

  /**
   * R9-19: a refresh that wrote the connection row changed what the allowlist
   * ConfigMap must say, and mcp-host and HCC read the ConfigMap, not Postgres.
   * Redemption republishes before it fails, as the refresh route does.
   */
  function persistedRejection() {
    return async () => {
      throw new GrokSubscriptionOAuthError('reauth_required', 'refresh token was rejected', {
        persistedConnectionStatus: true,
      })
    }
  }

  async function mutationWriteFailures(): Promise<number> {
    const metric = await llmAllowlistConfigMapWriteFailuresTotal.get()
    return metric.values.find(v => v.labels.phase === 'mutation')?.value ?? 0
  }

  it('R9-19 republishes the allowlist before failing when the refresh persisted a status', async () => {
    const h = arrange({ ensureFresh: persistedRejection() })

    await expect(h.run()).rejects.toMatchObject({
      name: 'GrokProviderAttemptRedeemError',
      code: 'no_grant',
    })

    expect(h.ensureFresh).toHaveBeenCalledWith('team-grok')
    expect(h.publishAllowlist).toHaveBeenCalledTimes(1)
    expect(h.withTransaction).not.toHaveBeenCalled()
  })

  it('R9-19 does not republish when the refresh failed without writing the row', async () => {
    const h = arrange({
      ensureFresh: async () => {
        throw new GrokSubscriptionOAuthError('provider_unavailable', 'refresh failed')
      },
    })

    await expect(h.run()).rejects.toMatchObject({ code: 'provider_unavailable' })

    // Liveness witness: the refresh ran and threw, so the missing publish is a
    // decision and not a path that never reached the catch.
    expect(h.ensureFresh).toHaveBeenCalledTimes(1)
    await expect(h.ensureFresh?.mock.results[0]?.value as Promise<void>).rejects.toBeInstanceOf(
      GrokSubscriptionOAuthError
    )
    expect(h.publishAllowlist).not.toHaveBeenCalled()
  })

  it('R9-19 counts a failed republish and still fails the redemption with its own code', async () => {
    const before = await mutationWriteFailures()
    const h = arrange({
      ensureFresh: persistedRejection(),
      publishAllowlist: async () => {
        throw new Error('configmap write refused')
      },
    })

    await expect(h.run()).rejects.toMatchObject({
      name: 'GrokProviderAttemptRedeemError',
      code: 'no_grant',
    })

    expect(h.publishAllowlist).toHaveBeenCalledTimes(1)
    expect(await mutationWriteFailures()).toBe(before + 1)
  })

  it('rethrows a non-OAuth refresh failure unchanged', async () => {
    const boom = new Error('socket hang up')
    const h = arrange({
      ensureFresh: async () => {
        throw boom
      },
    })
    await expect(h.run()).rejects.toBe(boom)
    expect(h.markRedeemed).not.toHaveBeenCalled()
  })

  const inTransaction: Array<[string, Scenario, string]> = [
    ['a ticket row that disappeared', { locked: null }, 'ticket_invalid'],
    [
      'a ticket redeemed concurrently',
      { locked: { ...ISSUED_TICKET, status: 'redeemed' } },
      'ticket_replayed',
    ],
    [
      'an expired ticket',
      { locked: { ...ISSUED_TICKET, expiresAt: new Date(Date.now() - 1) } },
      'ticket_expired',
    ],
    ['an attempt that vanished', { txAttempt: null }, 'ticket_invalid'],
    [
      'a non-authorized attempt',
      { txAttempt: attemptRow({ status: 'redeemed' }) },
      'ticket_invalid',
    ],
    [
      'a provider swap inside the transaction',
      { txAttempt: attemptRow({ provider: 'codex-subscription' }) },
      'ticket_invalid',
    ],
    ...(
      [
        ['requestHash', 'f'.repeat(64)],
        ['model', 'grok-other'],
        ['hostRef', 'other-host'],
        ['invocationId', 'invocation-2'],
        ['attemptGeneration', 2],
        ['policyHash', 'c'.repeat(64)],
        ['policyRevision', 5],
        ['budgetReservationId', 'res-other'],
        ['connectionRevision', 4],
      ] as const
    ).map(([field, value]): [string, Scenario, string] => [
      `${field} drift`,
      { txAttempt: attemptRow({ [field]: value }) },
      'no_grant',
    ]),
    ['a connection row that is gone', { connection: null }, 'connection_unavailable'],
    [
      'a connection needing reauth',
      { connection: liveConnection({ status: 'reauth_required' }) },
      'connection_unavailable',
    ],
    [
      'a revoked connection',
      { connection: liveConnection({ revokedAt: new Date() }) },
      'connection_unavailable',
    ],
    [
      'a credential revision change',
      { connection: liveConnection({ credentialRevision: CLAIMS.connectionRevision + 1 }) },
      'connection_unavailable',
    ],
  ]

  it.each(inTransaction)(
    'rejects %s before consuming the ticket',
    async (_label, scenario, code) => {
      const h = arrange(scenario)
      await expect(h.run()).rejects.toMatchObject({ name: 'GrokProviderAttemptRedeemError', code })
      expect(h.withTransaction).toHaveBeenCalledTimes(1)
      expect(h.markRedeemed).not.toHaveBeenCalled()
      expect(h.loadSecrets).not.toHaveBeenCalled()
    }
  )

  it('reports a lost consume race as ticket_replayed without loading secrets', async () => {
    const h = arrange({ markRedeemed: false })
    await expect(h.run()).rejects.toMatchObject({ code: 'ticket_replayed' })
    expect(h.loadSecrets).not.toHaveBeenCalled()
  })

  it.each([
    ['no secrets row', null],
    [
      'a null access token',
      {
        refreshToken: 'refresh-secret',
        accessToken: null,
        accessTokenExpiresAt: null,
        credentialRevision: CLAIMS.connectionRevision,
      },
    ],
  ])('throws inside the transaction for %s so the consume rolls back', async (_label, secrets) => {
    const h = arrange({ secrets })
    let rolledBack = false
    h.withTransaction.mockImplementation(async work => {
      try {
        return await work({ query: vi.fn() })
      } catch (err) {
        rolledBack = true
        throw err
      }
    })
    await expect(h.run()).rejects.toMatchObject({ code: 'connection_unavailable' })
    expect(h.markRedeemed).toHaveBeenCalledTimes(1)
    expect(rolledBack).toBe(true)
  })
})