Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.0 KB
Raw Blame
/**
 * T1 for the subscription catalog reconciliation cron.
 *
 * The unit suite proves the tick's decision table against doubles. This one
 * proves the half the doubles cannot: that a tick driven through the real
 * broker port lands rows in `grok_catalog_models`, rebuilds the
 * `grok-subscription` union in `llm_allowed_models`, stale-flags a model that
 * vanished upstream, and stays idempotent across identical ticks.
 *
 * Everything below the xAI network boundary is real — OAuth refresh-token
 * rotation, secret encryption, the revision-fenced outcome write, the
 * transaction and the union rebuild. Only two things are doubled, and both are
 * systems outside this process: xAI's HTTP endpoints (`GrokCatalogTransport`
 * for the models call, `fetchFn` for the token call) and the Kubernetes
 * ConfigMap writer.
 */
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes } from 'node:crypto'
import { Pool } from 'pg'
import { type DbClient, initDb } from '../src/db.js'
import { deriveOAuthEncryptionKey } from '../src/oauth/encryption.js'
import {
  type GrokCatalogTransport,
  type GrokTransactionRunner,
} from '../src/services/grokSubscriptionCatalog.js'
import {
  createNamedGrokSubscriptionConnection,
  getSafeGrokSubscriptionConnection,
  listLiveGrokSubscriptionConnections,
} from '../src/services/grokSubscriptionConnection.js'
import {
  GROK_OAUTH_DEVICE_URL,
  GROK_OAUTH_REVOKE_URL,
  GROK_OAUTH_TOKEN_URL,
  type GrokOAuthDeps,
  pollGrokDevice,
  runGrokCatalogSync,
  startGrokDeviceConnect,
} from '../src/services/grokSubscriptionOAuth.js'
import type { AllowedModelsConfigMapMaterializer } from '../src/services/llmAllowedModelsConfigMap.js'
import {
  type SubscriptionBrokerPort,
  filterAddressableGrokConnections,
  reconcileSubscriptionCatalogs,
} from '../src/services/subscriptionCatalogSyncCron.js'
import './realPostgres.requirement.ts'

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip
const KEY = deriveOAuthEncryptionKey('ab'.repeat(32))

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

function idTokenFor(subject: string): string {
  return `hdr.${Buffer.from(JSON.stringify({ sub: subject })).toString('base64url')}.sig`
}

function jsonResponse(status: number, body: Record<string, unknown>): Response {
  return { ok: status >= 200 && status < 300, status, json: async () => body } as Response
}

/**
 * The xAI account behind every exchange in this suite. One subject for the
 * device grant and for every later refresh, because they are one account: the
 * refresh recomputes the fingerprint and compares it with the one the grant
 * stored, so a subject that varied per exchange would read as an account swap
 * and flip the connection to `reauth_required` before the catalog is touched.
 */
const GRANT_SUBJECT = 'subject-cron-grok'

/**
 * A second xAI account, for the grant whose refresh is rejected.
 *
 * The subject must be stable across one grant and its refreshes, and it must
 * differ between two grants that are live at the same time: the fingerprint is
 * derived from the subject, and `persistGrantedTokens` refuses a second live
 * subscription on one account with `fingerprint_in_use`. The rejected grant is
 * created while the healthy one is still connected, so reusing `GRANT_SUBJECT`
 * here makes the connect throw before the tick under test ever runs.
 */
const REJECTED_GRANT_SUBJECT = 'subject-cron-grok-rejected'

/**
 * Simulated xAI device-authorization provider for the healthy cases. Its grant
 * is valid for an hour, well outside `ACCESS_TOKEN_REFRESH_SKEW_MS`, so a tick
 * using it finds a fresh access token and goes straight to the catalog:
 * `ensureFreshGrokAccessToken` returns before `rotateLockedRefresh`, and this
 * token endpoint is never called a second time. That is the ordinary steady
 * state, and it is deliberately NOT the rotation path — see
 * `rejectingRefreshGrokProvider` for the tick that does rotate.
 */
function simulatedGrokProvider(): typeof fetch {
  let issued = 0
  return (async (url: string | URL) => {
    const target = String(url)
    if (target === GROK_OAUTH_DEVICE_URL) {
      issued += 1
      return jsonResponse(200, {
        device_code: `device-code-${issued}-${randomBytes(4).toString('hex')}`,
        user_code: 'ABCD-EFGH',
        verification_uri: 'https://auth.x.ai/activate',
        expires_in: 900,
        interval: 5,
      })
    }
    if (target === GROK_OAUTH_TOKEN_URL) {
      return jsonResponse(200, {
        access_token: `access-${randomBytes(4).toString('hex')}`,
        refresh_token: `refresh-${randomBytes(4).toString('hex')}`,
        expires_in: 3600,
        id_token: idTokenFor(GRANT_SUBJECT),
      })
    }
    if (target === GROK_OAUTH_REVOKE_URL) return jsonResponse(200, {})
    return jsonResponse(404, {})
  }) as typeof fetch
}

/**
 * Simulated xAI provider whose grant expires inside the refresh skew and whose
 * refresh exchange is then rejected with `invalid_grant` — the state a real
 * deployment reaches when the operator revokes the app at xAI.
 *
 * The device grant and the refresh share one endpoint, so the first token call
 * is the grant and every later one is the rotation this suite's other provider
 * never reaches. `expires_in` of 60 seconds sits inside
 * `ACCESS_TOKEN_REFRESH_SKEW_MS` (5 minutes), which is what makes
 * `ensureFreshGrokAccessToken` rotate instead of returning early.
 *
 * This is the same HTTP seam the healthy cases use, not a new double: xAI's
 * endpoints are the boundary, so a rejected refresh needs no real subscription.
 */
function rejectingRefreshGrokProvider(): typeof fetch {
  let tokenCalls = 0
  return (async (url: string | URL) => {
    const target = String(url)
    if (target === GROK_OAUTH_DEVICE_URL) {
      return jsonResponse(200, {
        device_code: `device-code-reject-${randomBytes(4).toString('hex')}`,
        user_code: 'IJKL-MNOP',
        verification_uri: 'https://auth.x.ai/activate',
        expires_in: 900,
        interval: 5,
      })
    }
    if (target === GROK_OAUTH_TOKEN_URL) {
      tokenCalls += 1
      if (tokenCalls === 1) {
        return jsonResponse(200, {
          access_token: `access-${randomBytes(4).toString('hex')}`,
          refresh_token: `refresh-${randomBytes(4).toString('hex')}`,
          expires_in: 60,
          id_token: idTokenFor(REJECTED_GRANT_SUBJECT),
        })
      }
      return jsonResponse(400, { error: 'invalid_grant' })
    }
    if (target === GROK_OAUTH_REVOKE_URL) return jsonResponse(200, {})
    return jsonResponse(404, {})
  }) as typeof fetch
}

function readyTransport(models: Array<{ model: string; contextWindowTokens?: number }>): {
  transport: GrokCatalogTransport
  calls: number[]
} {
  const calls: number[] = []
  return {
    calls,
    transport: {
      listModels: async () => {
        calls.push(models.length)
        return { outcome: 'ready', models }
      },
    },
  }
}

type AllowedModelRow = {
  model: string
  enabled: boolean
  stale: boolean
  source: string
  vendor: string | null
  context_window_tokens: number | null
}

describeRealPostgres('Subscription catalog sync cron on real PostgreSQL', () => {
  const database = `subscription_catalog_cron_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? `postgresql://postgres@${['127', '0', '0', '1'].join('.')}/postgres`,
    database
  )
  let adminPool: Pool
  let pool: Pool

  const poolTransaction: GrokTransactionRunner = async work => {
    const client = await pool.connect()
    try {
      await client.query('BEGIN')
      const result = await work(client)
      await client.query('COMMIT')
      return result
    } catch (err) {
      await client.query('ROLLBACK').catch(() => undefined)
      throw err
    } finally {
      client.release()
    }
  }

  // `fetchFn` is a parameter because a provider that counts its own calls has to
  // be the SAME instance across the device grant and the later tick; a helper
  // that built a fresh one per call would reset that counter and hand the tick
  // a second successful grant instead of the rejection under test.
  function oauthDeps(
    connectionKey: string,
    db: DbClient = pool,
    fetchFn: typeof fetch = simulatedGrokProvider()
  ): GrokOAuthDeps {
    return {
      db,
      encryptionKey: KEY,
      fetchFn,
      clientId: 'b1a00492-073a-47ea-816f-4c329264a828',
      enabled: true,
      connectionKey,
      withTransaction: poolTransaction,
    }
  }

  /**
   * The production Grok port, rebuilt over this suite's pool. Every collaborator
   * except the transport is the one `createSubscriptionBrokerPorts` wires.
   */
  function grokPort(
    transport: GrokCatalogTransport,
    syncedKeys: string[] = [],
    fetchFn?: typeof fetch
  ): SubscriptionBrokerPort {
    return {
      broker: 'grok-subscription',
      enabled: true,
      listConnections: async () =>
        filterAddressableGrokConnections(await listLiveGrokSubscriptionConnections(pool)),
      syncCatalog: key => {
        syncedKeys.push(key)
        return runGrokCatalogSync(oauthDeps(key, pool, fetchFn), key, transport)
      },
    }
  }

  function recordingMaterializer(): {
    materializer: AllowedModelsConfigMapMaterializer
    calls: number[]
  } {
    const calls: number[] = []
    return {
      calls,
      materializer: {
        materialize: async () => {
          calls.push(Date.now())
        },
      },
    }
  }

  async function allowedModels(): Promise<AllowedModelRow[]> {
    const result = await pool.query<AllowedModelRow>(
      `SELECT model, enabled, stale, source, vendor, context_window_tokens
         FROM llm_allowed_models
        WHERE provider = 'grok-subscription'
        ORDER BY model ASC`
    )
    return result.rows
  }

  async function catalogModels(
    connectionId: string
  ): Promise<Array<{ model: string; stale: boolean; enabled: boolean }>> {
    const result = await pool.query<{ model: string; stale: boolean; enabled: boolean }>(
      `SELECT model, stale, enabled
         FROM grok_catalog_models
        WHERE connection_id = $1
        ORDER BY model ASC`,
      [connectionId]
    )
    return result.rows
  }

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    pool = new Pool({ connectionString })
    await initDb({ connect: () => pool.connect() })
  }, 60_000)

  afterAll(async () => {
    try {
      await pool?.end()
      if (adminPool) {
        await adminPool
          .query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
          .catch(() => undefined)
      }
    } finally {
      await adminPool?.end()
    }
  })

  it('a tick writes the broker catalog and rebuilds the grok-subscription allowlist', async () => {
    const key = 'cron-grok-primary'
    // The grant is created through the device flow the operator uses, not by
    // writing credential rows: that is what leaves a fingerprint the cron's
    // own refresh can match, and it is the state a real tick finds.
    const started = await startGrokDeviceConnect(oauthDeps(key), 'connect')
    const polled = await pollGrokDevice(oauthDeps(key), started.state)
    expect(polled.status).toBe('connected')
    const created = await getSafeGrokSubscriptionConnection(pool, key)
    expect(created).toMatchObject({ status: 'connected', catalogStatus: 'never_synced' })
    const { transport, calls } = readyTransport([
      { model: 'grok-4-fast-reasoning', contextWindowTokens: 2_000_000 },
      { model: 'grok-4' },
    ])
    const { materializer, calls: published } = recordingMaterializer()

    const result = await reconcileSubscriptionCatalogs({
      brokers: [grokPort(transport)],
      materializer,
    })

    expect(result).toEqual({
      synced: 1,
      degraded: 0,
      raced: 0,
      failed: 0,
      skipped: 0,
      published: 'published',
    })
    expect(calls).toEqual([2])
    expect(published).toHaveLength(1)
    expect(await catalogModels(created!.id)).toEqual([
      { model: 'grok-4', stale: false, enabled: true },
      { model: 'grok-4-fast-reasoning', stale: false, enabled: true },
    ])
    expect(await allowedModels()).toEqual([
      {
        model: 'grok-4',
        enabled: true,
        stale: false,
        source: 'discovery',
        vendor: 'xAI',
        context_window_tokens: null,
      },
      {
        model: 'grok-4-fast-reasoning',
        enabled: true,
        stale: false,
        source: 'discovery',
        vendor: 'xAI',
        context_window_tokens: 2_000_000,
      },
    ])
  })

  it('an identical second tick records a fresh outcome without duplicating rows', async () => {
    const key = 'cron-grok-primary'
    const before = await getSafeGrokSubscriptionConnection(pool, key)
    expect(before).toMatchObject({ catalogStatus: 'ready' })
    const { transport } = readyTransport([
      { model: 'grok-4-fast-reasoning', contextWindowTokens: 2_000_000 },
      { model: 'grok-4' },
    ])
    const { materializer } = recordingMaterializer()

    const result = await reconcileSubscriptionCatalogs({
      brokers: [grokPort(transport)],
      materializer,
    })

    expect(result).toMatchObject({ synced: 1, failed: 0, published: 'published' })
    // Liveness witness: the tick really re-ran the fenced write. Without it,
    // "the rows did not change" would also hold for a tick that did nothing.
    const after = await getSafeGrokSubscriptionConnection(pool, key)
    expect(after?.catalogRevision).toBe((before?.catalogRevision ?? 0) + 1)
    expect((await allowedModels()).map(row => row.model)).toEqual([
      'grok-4',
      'grok-4-fast-reasoning',
    ])
  })

  it('a model that vanished upstream is stale-flagged in the catalog and disabled in the union', async () => {
    const key = 'cron-grok-primary'
    const connection = await getSafeGrokSubscriptionConnection(pool, key)
    expect(connection).not.toBeNull()
    const { transport, calls } = readyTransport([{ model: 'grok-4' }])
    const { materializer } = recordingMaterializer()

    const result = await reconcileSubscriptionCatalogs({
      brokers: [grokPort(transport)],
      materializer,
    })

    expect(result).toMatchObject({ synced: 1, failed: 0 })
    expect(calls).toEqual([1])
    expect(await catalogModels(connection!.id)).toEqual([
      { model: 'grok-4', stale: false, enabled: true },
      { model: 'grok-4-fast-reasoning', stale: true, enabled: true },
    ])
    expect(await allowedModels()).toEqual([
      expect.objectContaining({ model: 'grok-4', enabled: true, stale: false }),
      expect.objectContaining({ model: 'grok-4-fast-reasoning', enabled: false, stale: true }),
    ])
  })

  it('a grant that is not connected is skipped and never reaches the broker', async () => {
    const idle = 'cron-grok-idle'
    await createNamedGrokSubscriptionConnection(pool, {
      connectionKey: idle,
      displayName: 'idle',
    })
    const { transport } = readyTransport([{ model: 'grok-4' }])
    const syncedKeys: string[] = []
    const { materializer } = recordingMaterializer()

    const result = await reconcileSubscriptionCatalogs({
      brokers: [grokPort(transport, syncedKeys)],
      materializer,
    })

    // The negative claim is `idle` never being synced. Its witness is the
    // connected grant in the same listing that WAS synced: a port that
    // returned nothing at all would satisfy the exclusion for free.
    expect(syncedKeys).toEqual(['cron-grok-primary'])
    expect(result).toMatchObject({ synced: 1, skipped: 1, failed: 0 })
    expect(await getSafeGrokSubscriptionConnection(pool, idle)).toMatchObject({
      status: 'disconnected',
      catalogStatus: 'never_synced',
    })
  })

  it('a rejected refresh marks the row and still counts as a recorded outcome', async () => {
    const rejected = 'cron-grok-rejected'
    // One provider instance for the grant AND the tick: the rejection is its
    // second token call, so a fresh instance per call would hand the tick
    // another successful grant.
    const provider = rejectingRefreshGrokProvider()
    const started = await startGrokDeviceConnect(oauthDeps(rejected, pool, provider), 'connect')
    const polled = await pollGrokDevice(oauthDeps(rejected, pool, provider), started.state)
    expect(polled.status).toBe('connected')
    const { transport, calls } = readyTransport([{ model: 'grok-4' }])
    const syncedKeys: string[] = []
    const { materializer, calls: published } = recordingMaterializer()

    const result = await reconcileSubscriptionCatalogs({
      brokers: [grokPort(transport, syncedKeys, provider)],
      materializer,
    })

    // `degraded` is the load-bearing count, not a detail of the tally. A
    // rejected refresh returns `catalogStatus: 'never_synced'`, so the only
    // thing that keeps it out of `failed` is `persisted: true` travelling from
    // the error, through `runGrokCatalogSync`, into `syncOutcomeChangedTheRow`.
    // Sever any link in that chain and this line reads `degraded: 0, failed: 1`
    // — which is the whole mechanism this feature exists for, observed at the
    // level where it runs rather than asserted on a hand-built outcome.
    expect(result).toMatchObject({ synced: 1, degraded: 1, failed: 0, raced: 0 })
    // The business witness in Postgres: the connection really was written
    // before the throw, which is what makes the stale ConfigMap a problem.
    expect(await getSafeGrokSubscriptionConnection(pool, rejected)).toMatchObject({
      status: 'reauth_required',
      catalogStatus: 'never_synced',
    })
    // Liveness witnesses for the negative: the healthy grant in the same
    // listing was synced and its catalog really was read, so `calls` holding a
    // single entry means the rejected grant never reached the transport — not
    // that the tick did nothing.
    expect(syncedKeys).toContain(rejected)
    expect(syncedKeys).toContain('cron-grok-primary')
    expect(calls).toEqual([1])
    expect(published).toHaveLength(1)
  })
})