Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.7 KB
Raw Blame
/**
 * Subscription catalog reconciliation cron.
 *
 * The tick reconciles every live, connected subscription grant against its
 * broker's catalog under a SESSION advisory lock, so replicas dedup instead of
 * piling up redundant upstream calls. This suite pins the hazards that make the
 * difference between a reconciler and an abuse vector: the lock lifecycle, the
 * skip rules (a disabled broker, a grant that is not `connected`), failure
 * isolation per connection, and the single ConfigMap publish — which runs on
 * every tick, including one that recorded nothing, because that is the tick
 * after a publish that threw and the rows it changed are skipped by status.
 *
 * Deps are injected: no database, no cluster, no upstream. The production
 * wiring is asserted where it belongs — the route suite for the endpoint and
 * the real-Postgres suite for the schema.
 */
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { AllowedModelsConfigMapMaterializer } from '../llmAllowedModelsConfigMap.js'
import {
  type SubscriptionBrokerPort,
  type SubscriptionCatalogSyncOutcome,
  filterAddressableGrokConnections,
  reconcileSubscriptionCatalogs,
  runSubscriptionCatalogSyncTick,
  startSubscriptionCatalogSyncCron,
  stopSubscriptionCatalogSyncCron,
} from '../subscriptionCatalogSyncCron.js'

vi.mock('../../observability/logger.js', () => ({
  rootLogger: { child: () => ({ info() {}, warn() {}, error() {}, debug() {} }) },
}))

const READY: SubscriptionCatalogSyncOutcome = { ok: true, catalogStatus: 'ready' }
/** Recorded, but not ready: the row now carries `auth-rejected`. */
const AUTH_REJECTED: SubscriptionCatalogSyncOutcome = { ok: false, catalogStatus: 'auth-rejected' }
/** Nothing recorded: a concurrent writer won the revision fence. */
const STALE: SubscriptionCatalogSyncOutcome = {
  ok: false,
  catalogStatus: 'never_synced',
  reason: 'stale_revision',
}
/**
 * The write landed and the sync still reports `never_synced`.
 * `markGrokRefreshSubjectMismatch` sets `status = 'reauth_required'` and THEN
 * throws, so the catalog was never synced while the CONNECTION row changed.
 * `llmAllowedModelsConfigMap` maps that status into the ConfigMap, so the
 * publish is owed — `never_synced` alone cannot decide it.
 */
const PERSISTED_REAUTH: SubscriptionCatalogSyncOutcome = {
  ok: false,
  catalogStatus: 'never_synced',
  reason: 'reauth_required',
  persisted: true,
}

/**
 * A fake pool sharing one advisory-lock state across every client it hands out
 * — the real cross-replica contract. `calls` records the lock lifecycle so a
 * leaked SESSION lock is visible rather than inferred.
 */
function makeLockPool(
  shared: { held: boolean },
  calls: string[],
  opts: { failUnlock?: boolean } = {}
) {
  const releaseArgs: Array<Error | boolean | undefined> = []
  return {
    releaseArgs,
    connect: vi.fn(async () => {
      const query = vi.fn(async (sql: string) => {
        if (/pg_try_advisory_lock/.test(sql)) {
          const acquired = !shared.held
          if (acquired) shared.held = true
          calls.push(acquired ? 'acquire:ok' : 'acquire:skip')
          return { rows: [{ acquired }], rowCount: 1 }
        }
        if (/pg_advisory_unlock/.test(sql)) {
          if (opts.failUnlock) {
            calls.push('unlock:throw')
            throw new Error('unlock failed')
          }
          shared.held = false
          calls.push('unlock')
          return { rows: [{ pg_advisory_unlock: true }], rowCount: 1 }
        }
        return { rows: [], rowCount: 0 }
      })
      const release = vi.fn((destroy?: Error | boolean) => {
        releaseArgs.push(destroy)
        if (destroy) {
          shared.held = false
          calls.push('release:destroy')
        } else {
          calls.push('release')
        }
      })
      return { query, release }
    }),
  }
}

function makePort(
  broker: SubscriptionBrokerPort['broker'],
  rows: Array<{ connectionKey: string; status: string }>,
  sync: (connectionKey: string) => Promise<SubscriptionCatalogSyncOutcome>,
  enabled = true
): SubscriptionBrokerPort & {
  listConnections: ReturnType<typeof vi.fn>
  syncCatalog: ReturnType<typeof vi.fn>
} {
  return {
    broker,
    enabled,
    listConnections: vi.fn(async () => rows),
    syncCatalog: vi.fn(sync),
  }
}

function makeMaterializer(): AllowedModelsConfigMapMaterializer & {
  materialize: ReturnType<typeof vi.fn>
} {
  return { materialize: vi.fn(async () => undefined) }
}

afterEach(() => {
  stopSubscriptionCatalogSyncCron()
  vi.useRealTimers()
  vi.unstubAllGlobals()
})

describe('subscription catalog sync tick — advisory lock', () => {
  it('runs the reconciliation and releases the lock when it acquires it', async () => {
    const shared = { held: false }
    const calls: string[] = []
    const connector = makeLockPool(shared, calls)
    const sync = vi.fn(async () => ({
      synced: 1,
      degraded: 0,
      raced: 0,
      failed: 0,
      skipped: 0,
      published: 'published' as const,
    }))

    const result = await runSubscriptionCatalogSyncTick({ connector, sync })

    expect(result).toEqual({ skippedLock: false, ran: true, errored: false })
    expect(sync).toHaveBeenCalledTimes(1)
    expect(calls).toEqual(['acquire:ok', 'unlock', 'release'])
    expect(shared.held).toBe(false)
  })

  it('skips the tick without syncing when another replica holds the lock', async () => {
    const shared = { held: true }
    const calls: string[] = []
    const connector = makeLockPool(shared, calls)
    const sync = vi.fn()

    const result = await runSubscriptionCatalogSyncTick({ connector, sync })

    expect(result).toEqual({ skippedLock: true, ran: false, errored: false })
    // Liveness witness: the tick did reach the lock and was refused it, so the
    // un-called sync is a skip and not an unreached code path.
    expect(calls).toEqual(['acquire:skip', 'release'])
    expect(sync).not.toHaveBeenCalled()
    // The other replica's lock is untouched.
    expect(shared.held).toBe(true)
  })

  it('destroys the connection when the unlock itself fails, so the session lock cannot leak', async () => {
    const shared = { held: false }
    const calls: string[] = []
    const connector = makeLockPool(shared, calls, { failUnlock: true })
    const sync = vi.fn(async () => ({
      synced: 0,
      degraded: 0,
      raced: 0,
      failed: 0,
      skipped: 0,
      published: 'skipped' as const,
    }))

    await runSubscriptionCatalogSyncTick({ connector, sync })

    expect(calls).toEqual(['acquire:ok', 'unlock:throw', 'release:destroy'])
    expect(connector.releaseArgs).toHaveLength(1)
    expect(connector.releaseArgs[0]).toBeInstanceOf(Error)
    expect(shared.held).toBe(false)
  })

  it('reports a failing reconciliation without throwing, and still releases the lock', async () => {
    const shared = { held: false }
    const calls: string[] = []
    const connector = makeLockPool(shared, calls)
    const sync = vi.fn(async () => {
      throw new Error('reconciliation exploded')
    })

    const result = await runSubscriptionCatalogSyncTick({ connector, sync })

    expect(result).toEqual({ skippedLock: false, ran: false, errored: true })
    expect(calls).toEqual(['acquire:ok', 'unlock', 'release'])
    expect(shared.held).toBe(false)
  })
})

describe('subscription catalog reconciliation — skip rules', () => {
  it('skips a broker whose deployment gate is off without listing its connections', async () => {
    const codex = makePort(
      'codex-subscription',
      [{ connectionKey: 'a', status: 'connected' }],
      async () => READY
    )
    const grok = makePort(
      'grok-subscription',
      [{ connectionKey: 'b', status: 'connected' }],
      async () => READY,
      false
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex, grok], materializer })

    // Liveness witness: the enabled broker ran, so the loop was entered and the
    // disabled broker was skipped by its gate, not by an unreached path.
    expect(codex.syncCatalog).toHaveBeenCalledWith('a')
    expect(grok.listConnections).not.toHaveBeenCalled()
    expect(grok.syncCatalog).not.toHaveBeenCalled()
    expect(result.synced).toBe(1)
  })

  it('syncs only grants whose status is connected', async () => {
    const rows = [
      { connectionKey: 'disconnected-key', status: 'disconnected' },
      { connectionKey: 'connecting-key', status: 'connecting' },
      { connectionKey: 'reauth-key', status: 'reauth_required' },
      { connectionKey: 'connected-key', status: 'connected' },
    ]
    const grok = makePort('grok-subscription', rows, async () => READY)
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [grok], materializer })

    // Liveness witness: the connected sibling in the SAME list was synced, so
    // the three unsynced rows were filtered, not merely never reached.
    expect(grok.syncCatalog.mock.calls.map(call => call[0])).toEqual(['connected-key'])
    expect(result.skipped).toBe(3)
    expect(result.synced).toBe(1)
  })

  it('drops archived Grok tombstone keys that sit outside the key grammar', () => {
    // The key grammar is a DNS label (`grokSubscriptionSchema.ts:4`), so the
    // archive suffix `~revoked~<id>` puts a tombstone outside it by construction.
    const rows = [
      { connectionKey: 'grok-primary', status: 'connected' },
      { connectionKey: 'grok-primary~revoked~91f3', status: 'revoked' },
    ]

    expect(filterAddressableGrokConnections(rows)).toEqual([
      { connectionKey: 'grok-primary', status: 'connected' },
    ])
  })
})

describe('subscription catalog reconciliation — isolation and outcomes', () => {
  it('keeps going after one connection throws, and counts it as failed', async () => {
    const rows = [
      { connectionKey: 'first', status: 'connected' },
      { connectionKey: 'boom', status: 'connected' },
      { connectionKey: 'last', status: 'connected' },
    ]
    const codex = makePort('codex-subscription', rows, async key => {
      if (key === 'boom') throw new Error('upstream refused the connection')
      return READY
    })
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex], materializer })

    expect(codex.syncCatalog.mock.calls.map(call => call[0])).toEqual(['first', 'boom', 'last'])
    expect(result).toMatchObject({ synced: 2, failed: 1, raced: 0, degraded: 0 })
  })

  it('counts a raced revision or a held refresh lock as transient, not as a failure', async () => {
    const rows = [
      { connectionKey: 'raced', status: 'connected' },
      { connectionKey: 'locked', status: 'connected' },
    ]
    const grok = makePort('grok-subscription', rows, async key =>
      key === 'raced'
        ? STALE
        : { ok: false, catalogStatus: 'never_synced', reason: 'refresh_in_flight' }
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [grok], materializer })

    expect(result).toMatchObject({ raced: 2, failed: 0, synced: 0, degraded: 0 })
  })
})

describe('subscription catalog reconciliation — ConfigMap publish', () => {
  it('publishes once after the loop, not once per connection', async () => {
    const rows = [
      { connectionKey: 'one', status: 'connected' },
      { connectionKey: 'two', status: 'connected' },
    ]
    const codex = makePort('codex-subscription', rows, async () => READY)
    const grok = makePort(
      'grok-subscription',
      [{ connectionKey: 'three', status: 'connected' }],
      async () => READY
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex, grok], materializer })

    expect(result.synced).toBe(3)
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
    expect(result.published).toBe('published')
  })

  it('publishes a recorded NON-ready outcome, because the row now carries that status', async () => {
    const codex = makePort(
      'codex-subscription',
      [{ connectionKey: 'rejected', status: 'connected' }],
      async () => AUTH_REJECTED
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex], materializer })

    expect(result).toMatchObject({ synced: 0, degraded: 1, published: 'published' })
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
  })

  it('publishes although nothing recorded an outcome, so a failed publish converges', async () => {
    const codex = makePort(
      'codex-subscription',
      [
        { connectionKey: 'skipped', status: 'reauth_required' },
        { connectionKey: 'raced', status: 'connected' },
      ],
      async () => STALE
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex], materializer })

    // Liveness witness: the loop ran and reached the broker — one row was
    // skipped by status and the other really was synced and raced — so the
    // publish under test happened after a tick that wrote nothing, which is
    // the whole point. This shape is the tick AFTER a publish that threw: the
    // rows it changed are `reauth_required` by then and get skipped here, so
    // gating the publish on "something was recorded" would strand the stale
    // ConfigMap forever.
    expect(codex.listConnections).toHaveBeenCalledTimes(1)
    expect(codex.syncCatalog).toHaveBeenCalledWith('raced')
    expect(result).toMatchObject({ skipped: 1, raced: 1 })
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
    expect(result.published).toBe('published')
  })

  it('reports skipped when no materializer is wired, without calling one', async () => {
    const codex = makePort(
      'codex-subscription',
      [{ connectionKey: 'one', status: 'connected' }],
      async () => READY
    )

    const result = await reconcileSubscriptionCatalogs({
      brokers: [codex],
      materializer: undefined,
    })

    // Liveness witness for the negative: the sync really ran, so `skipped`
    // here is the absent writer and not an abandoned tick.
    expect(codex.syncCatalog).toHaveBeenCalledWith('one')
    expect(result).toMatchObject({ synced: 1, published: 'skipped' })
  })

  it('publishes when the sync persisted a status although it reports never_synced', async () => {
    const grok = makePort(
      'grok-subscription',
      [{ connectionKey: 'rejected', status: 'connected' }],
      async () => PERSISTED_REAUTH
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [grok], materializer })

    // Liveness witness: the loop reached the broker and ran the sync, so the
    // publish below is the rule under test rather than an unvisited path.
    expect(grok.syncCatalog).toHaveBeenCalledWith('rejected')
    // The row carries `reauth_required` now. Counting this as `failed` and
    // skipping the publish leaves mcp-host and HCC serving `connected` with the
    // full model list until an unrelated grant mutation republishes.
    expect(result).toMatchObject({ degraded: 1, failed: 0, published: 'published' })
    expect(materializer.materialize).toHaveBeenCalledTimes(1)
  })

  it('contains a broker whose connection listing fails, so the other broker still runs', async () => {
    const codex = makePort('codex-subscription', [], async () => READY)
    codex.listConnections.mockRejectedValueOnce(new Error('connection listing failed'))
    const grok = makePort(
      'grok-subscription',
      [{ connectionKey: 'team-grok', status: 'connected' }],
      async () => READY
    )
    const materializer = makeMaterializer()

    const result = await reconcileSubscriptionCatalogs({ brokers: [codex, grok], materializer })

    // `listConnections` runs OUTSIDE the per-connection try, so without its own
    // guard this rejection escapes the broker loop and the whole tick is lost —
    // including every Grok connection, which has nothing to do with it.
    expect(codex.listConnections).toHaveBeenCalledTimes(1)
    expect(result).toMatchObject({ failed: 1, synced: 1, published: 'published' })
    expect(grok.syncCatalog).toHaveBeenCalledWith('team-grok')
  })

  it('records the tick as failed when the publish itself fails, without throwing', async () => {
    const codex = makePort(
      'codex-subscription',
      [{ connectionKey: 'one', status: 'connected' }],
      async () => READY
    )
    const materializer = makeMaterializer()
    materializer.materialize.mockRejectedValueOnce(new Error('configmap write failed'))

    await expect(reconcileSubscriptionCatalogs({ brokers: [codex], materializer })).rejects.toThrow(
      /configmap write failed/
    )

    // The catalog rows were already written; only the publish failed. The tick
    // surfaces it (runSubscriptionCatalogSyncTick turns it into errored:true)
    // instead of reporting a clean reconciliation.
    expect(codex.syncCatalog).toHaveBeenCalledTimes(1)
  })
})

describe('subscription catalog cron scheduling', () => {
  it('runs a first jittered tick and then one per interval, with unref()d handles', async () => {
    vi.useFakeTimers()
    // The title claims unref(), so the test has to check it. An un-unref'd
    // handle keeps the event loop alive and control-api stops exiting on
    // SIGTERM — a failure that never shows up in a test that only counts ticks.
    const unrefed: string[] = []
    const realSetTimeout = globalThis.setTimeout
    const realSetInterval = globalThis.setInterval
    const recordUnref = <T extends { unref: () => T }>(handle: T, label: string): T => {
      const original = handle.unref.bind(handle)
      handle.unref = () => {
        unrefed.push(label)
        return original()
      }
      return handle
    }
    vi.stubGlobal('setTimeout', ((fn: () => void, ms?: number) =>
      recordUnref(realSetTimeout(fn, ms), 'firstRun')) as unknown as typeof setTimeout)
    vi.stubGlobal('setInterval', ((fn: () => void, ms?: number) =>
      recordUnref(realSetInterval(fn, ms), 'interval')) as unknown as typeof setInterval)

    const shared = { held: false }
    const calls: string[] = []
    const connector = makeLockPool(shared, calls)
    const sync = vi.fn(async () => ({
      synced: 0,
      degraded: 0,
      raced: 0,
      failed: 0,
      skipped: 0,
      published: 'skipped' as const,
    }))

    startSubscriptionCatalogSyncCron({ connector, sync }, 60_000)

    await vi.advanceTimersByTimeAsync(5_000)
    expect(sync).toHaveBeenCalledTimes(1)
    // The first-run handle is unref'd before the timer fires; the interval
    // handle only exists after it.
    expect(unrefed).toEqual(['firstRun', 'interval'])

    await vi.advanceTimersByTimeAsync(60_000)
    expect(sync).toHaveBeenCalledTimes(2)

    stopSubscriptionCatalogSyncCron()
    await vi.advanceTimersByTimeAsync(120_000)
    expect(sync).toHaveBeenCalledTimes(2)
  })
})