Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
32.5 KB
Raw Blame
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
  ExternalEgressConvergenceCoordinator,
  type ExternalEgressRetryHandle,
  type ExternalEgressWatchEventType,
} from './externalEgressConvergenceCoordinator'
import { externalEgressRetriesAtCap } from './metrics'
import type { McpServerCRD } from './types'

function deferred<T = void>() {
  let resolve!: (value: T | PromiseLike<T>) => void
  let reject!: (reason?: unknown) => void
  const promise = new Promise<T>((resolvePromise, rejectPromise) => {
    resolve = resolvePromise
    reject = rejectPromise
  })
  return { promise, resolve, reject }
}

function server(
  name: string,
  generation = 1,
  options: { bindings?: boolean; namespace?: string } = {}
): McpServerCRD {
  return {
    name,
    namespace: options.namespace ?? 'mcp-server',
    uid: `${name}-uid`,
    generation,
    spec: {
      contextRef: 'default',
      image: `clerum/${name}:v${generation}`,
      transport: { type: 'streamableHttp', port: 3000 },
      egressBindings:
        options.bindings === false ? undefined : [{ dns: `${name}.example`, port: 443 }],
    },
  }
}

type CoordinatorOverrides = Partial<
  ConstructorParameters<typeof ExternalEgressConvergenceCoordinator>[0]
>

function coordinator(overrides: CoordinatorOverrides = {}) {
  const servers = new Map<string, McpServerCRD>()
  const mutate = vi.fn(async () => undefined)
  const replay = vi.fn(async () => undefined)
  const instance = new ExternalEgressConvergenceCoordinator({
    listServers: () => [...servers.values()],
    getCurrentServer: name => servers.get(name),
    inventoryAuthoritative: () => true,
    sameDesiredRevision: (left, right) =>
      left.uid === right.uid &&
      left.generation === right.generation &&
      JSON.stringify(left.spec) === JSON.stringify(right.spec),
    enqueue: async (_selected, work) => work(),
    mutate,
    replay,
    externalEgressRefreshMinTtlMs: () => Infinity,
    ...overrides,
  })
  return { instance, mutate, replay, servers }
}

async function flushMicrotasks(): Promise<void> {
  await Promise.resolve()
  await Promise.resolve()
  await Promise.resolve()
}

describe('ExternalEgressConvergenceCoordinator', () => {
  afterEach(() => {
    vi.useRealTimers()
    vi.restoreAllMocks()
  })

  it.each([
    ['presence', 'DELETED' as const, undefined],
    ['revision', 'MODIFIED' as const, server('redis-tools', 2)],
  ])(
    'lets an existing timer consume the latest %s intent and resets its attempt ladder',
    async (_change, nextType, nextServer) => {
      vi.useFakeTimers()
      const initial = server('redis-tools', 1)
      const state = new Map<string, McpServerCRD>([[initial.name, initial]])
      const replayed: Array<{ type: ExternalEgressWatchEventType; server: McpServerCRD }> = []
      let instance!: ExternalEgressConvergenceCoordinator
      instance = coordinator({
        getCurrentServer: name => state.get(name),
        replay: async (type, selected) => {
          replayed.push({ type, server: selected })
          instance.scheduleRetry(type, selected)
        },
      }).instance

      instance.scheduleRetry('ADDED', initial)
      if (nextServer) state.set(nextServer.name, nextServer)
      else state.delete(initial.name)
      instance.scheduleRetry(nextType, nextServer ?? initial)

      await vi.advanceTimersByTimeAsync(5000)
      expect(replayed).toEqual([
        {
          type: nextType,
          server: expect.objectContaining({
            generation: nextServer?.generation ?? initial.generation,
          }),
        },
      ])

      await vi.advanceTimersByTimeAsync(4999)
      expect(replayed).toHaveLength(1)
      await vi.advanceTimersByTimeAsync(1)
      expect(replayed).toHaveLength(2)
      instance.stop()
    }
  )

  it('continues retrying at the capped delay until the replay succeeds', async () => {
    vi.useFakeTimers()
    const selected = server('web-search')
    let instance!: ExternalEgressConvergenceCoordinator
    let attempts = 0
    instance = coordinator({
      getCurrentServer: () => selected,
      replay: async (type, current, retry) => {
        attempts += 1
        if (attempts < 5) {
          instance.scheduleRetry(type, current)
          return
        }
        retry.complete()
      },
    }).instance

    instance.scheduleRetry('ADDED', selected)
    await vi.advanceTimersByTimeAsync(5000)
    await vi.advanceTimersByTimeAsync(15000)
    await vi.advanceTimersByTimeAsync(30000)
    expect(attempts).toBe(3)
    await vi.advanceTimersByTimeAsync(29999)
    expect(attempts).toBe(3)
    await vi.advanceTimersByTimeAsync(1)
    expect(attempts).toBe(4)
    await vi.advanceTimersByTimeAsync(29999)
    expect(attempts).toBe(4)
    await vi.advanceTimersByTimeAsync(1)

    expect(attempts).toBe(5)
    await vi.advanceTimersByTimeAsync(30000)
    expect(attempts).toBe(5)
    instance.stop()
  })

  it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {
    vi.useFakeTimers()
    const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')
    const selected = server('web-search')
    let instance!: ExternalEgressConvergenceCoordinator
    let succeed = false
    instance = coordinator({
      getCurrentServer: () => selected,
      replay: async (type, current, retry) => {
        if (succeed) {
          retry.complete()
          return
        }
        instance.scheduleRetry(type, current)
      },
    }).instance

    instance.scheduleRetry('ADDED', selected)
    // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.
    await vi.advanceTimersByTimeAsync(5000)
    expect(setSpy).not.toHaveBeenCalled()

    // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.
    await vi.advanceTimersByTimeAsync(15000)
    expect(setSpy).toHaveBeenLastCalledWith(1)

    // Once the replay finally succeeds, the capped server is cleared back to 0.
    succeed = true
    await vi.advanceTimersByTimeAsync(30000)
    expect(setSpy).toHaveBeenLastCalledWith(0)

    instance.stop()
    setSpy.mockRestore()
  })

  it('never clears a pending full replay after an egress-only success', async () => {
    vi.useFakeTimers()
    const selected = server('web-search')
    const { instance, replay, servers } = coordinator()
    servers.set(selected.name, selected)

    instance.scheduleRetry('ADDED', selected)
    const retry = await instance.reconcile('MODIFIED', selected)
    expect(retry).toBeDefined()

    await vi.advanceTimersByTimeAsync(5000)
    expect(replay).toHaveBeenCalledOnce()
    instance.stop()
  })

  it('clears a retry only after its complete full-replay handle finishes', async () => {
    vi.useFakeTimers()
    const selected = server('web-search')
    let instance!: ExternalEgressConvergenceCoordinator
    const replays: ExternalEgressRetryHandle[] = []
    instance = coordinator({
      getCurrentServer: () => selected,
      replay: async (type, current, retry) => {
        replays.push(retry)
        const completion = await instance.reconcile(type, current, { retry })
        expect(completion).toBe(retry)
        retry.complete()
      },
    }).instance

    instance.scheduleRetry('ADDED', selected)
    await vi.advanceTimersByTimeAsync(5000)
    await vi.advanceTimersByTimeAsync(30000)

    expect(replays).toHaveLength(1)
    instance.stop()
  })

  it('prevents a stale retry handle from clearing a newer intent', async () => {
    vi.useFakeTimers()
    const first = server('web-search', 1)
    const second = server('web-search', 2)
    const state = new Map<string, McpServerCRD>([[first.name, first]])
    const handles: ExternalEgressRetryHandle[] = []
    const replayed: McpServerCRD[] = []
    const firstReplayStarted = deferred()
    const releaseFirstReplay = deferred()
    const { instance } = coordinator({
      getCurrentServer: name => state.get(name),
      replay: async (_type, current, retry) => {
        replayed.push(current)
        handles.push(retry)
        if (replayed.length === 1) {
          firstReplayStarted.resolve()
          await releaseFirstReplay.promise
        }
      },
    })

    instance.scheduleRetry('ADDED', first)
    await vi.advanceTimersByTimeAsync(5000)
    await firstReplayStarted.promise
    expect(handles).toHaveLength(1)

    state.set(second.name, second)
    instance.scheduleRetry('MODIFIED', second)
    handles[0].complete()
    releaseFirstReplay.resolve()
    await flushMicrotasks()
    await vi.advanceTimersByTimeAsync(5000)

    expect(replayed).toEqual([
      expect.objectContaining({ generation: 1 }),
      expect.objectContaining({ generation: 2 }),
    ])
    instance.stop()
  })

  it('keeps retry, resync, and startup concurrency lanes independent', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const retryRelease = deferred()
    const activeRetries = deferred()
    let retryCount = 0
    const resyncServer = server('resync-server')
    const startupServer = server('startup-server')
    const canonical = new Map<string, McpServerCRD>([
      [resyncServer.name, resyncServer],
      [startupServer.name, startupServer],
    ])
    const mutations: string[] = []
    const { instance } = coordinator({
      listServers: () => [resyncServer],
      getCurrentServer: name => canonical.get(name),
      mutate: async (_type, selected) => {
        mutations.push(selected.name)
      },
      replay: async () => {
        retryCount += 1
        if (retryCount === 10) activeRetries.resolve()
        await retryRelease.promise
      },
    })

    for (let index = 0; index < 10; index += 1) {
      const selected = server(`retry-${index}`, 1, { bindings: false })
      instance.scheduleRetry('ADDED', selected)
    }
    await vi.advanceTimersByTimeAsync(5000)
    await activeRetries.promise

    const resync = instance.runResync()
    const startup = instance.prepareStartupGates([startupServer])
    await Promise.all([resync, startup.waitFor(startupServer)])

    expect(mutations).toEqual(expect.arrayContaining(['resync-server', 'startup-server']))
    retryRelease.resolve()
    instance.stop()
  })

  it('queues at most one latest-wins retry job per key behind a saturated limiter', async () => {
    vi.useFakeTimers()
    const blockersRelease = deferred()
    const tenBlockersStarted = deferred()
    const finalReplayStarted = deferred()
    const initial = server('churning-server', 1)
    const second = server('churning-server', 2)
    const final = server('churning-server', 3)
    const state = new Map<string, McpServerCRD>([[initial.name, initial]])
    const targetEnqueues: McpServerCRD[] = []
    const targetReplays: McpServerCRD[] = []
    let blockerCount = 0
    const { instance } = coordinator({
      getCurrentServer: name => state.get(name),
      enqueue: async (selected, work) => {
        if (selected.name === initial.name) targetEnqueues.push(selected)
        await work()
      },
      replay: async (_type, selected, retry) => {
        if (selected.name.startsWith('limiter-blocker-')) {
          blockerCount += 1
          if (blockerCount === 10) tenBlockersStarted.resolve()
          await blockersRelease.promise
          return
        }
        targetReplays.push(selected)
        retry.complete()
        finalReplayStarted.resolve()
      },
    })

    for (let index = 0; index < 10; index += 1) {
      instance.scheduleRetry('DELETED', server(`limiter-blocker-${index}`))
    }
    await vi.advanceTimersByTimeAsync(5000)
    await tenBlockersStarted.promise

    instance.scheduleRetry('ADDED', initial)
    await vi.advanceTimersByTimeAsync(5000)
    state.set(second.name, second)
    instance.scheduleRetry('MODIFIED', second)
    await vi.advanceTimersByTimeAsync(5000)
    state.set(final.name, final)
    instance.scheduleRetry('MODIFIED', final)
    await vi.advanceTimersByTimeAsync(5000)

    expect(targetEnqueues).toHaveLength(0)
    blockersRelease.resolve()
    await finalReplayStarted.promise
    await flushMicrotasks()
    await flushMicrotasks()

    expect(targetEnqueues).toHaveLength(1)
    expect(targetReplays).toEqual([expect.objectContaining({ generation: 3 })])
    instance.stop()
  })

  it('re-arms a retry scheduled by an active replay only after that key leaves its slot', async () => {
    vi.useFakeTimers()
    const selected = server('slow-retry')
    const firstReplayStarted = deferred()
    const releaseFirstReplay = deferred()
    let instance!: ExternalEgressConvergenceCoordinator
    let replayCount = 0
    instance = coordinator({
      getCurrentServer: () => selected,
      replay: async (type, current, retry) => {
        replayCount += 1
        if (replayCount === 1) {
          instance.scheduleRetry(type, current)
          firstReplayStarted.resolve()
          await releaseFirstReplay.promise
          return
        }
        retry.complete()
      },
    }).instance

    instance.scheduleRetry('ADDED', selected)
    await vi.advanceTimersByTimeAsync(5000)
    await firstReplayStarted.promise

    await vi.advanceTimersByTimeAsync(15000)
    expect(replayCount).toBe(1)

    releaseFirstReplay.resolve()
    await flushMicrotasks()
    await vi.advanceTimersByTimeAsync(14999)
    expect(replayCount).toBe(1)
    await vi.advanceTimersByTimeAsync(1)
    expect(replayCount).toBe(2)
    instance.stop()
  })

  it('cancels a retry waiter on stop before limiter capacity becomes available', async () => {
    vi.useFakeTimers()
    const blockersRelease = deferred()
    const tenBlockersStarted = deferred()
    const queued = server('queued-before-stop')
    let blockerCount = 0
    const enqueuedTargets: string[] = []
    const { instance } = coordinator({
      enqueue: async (selected, work) => {
        if (selected.name === queued.name) enqueuedTargets.push(selected.name)
        await work()
      },
      replay: async (_type, selected) => {
        if (!selected.name.startsWith('stop-blocker-')) return
        blockerCount += 1
        if (blockerCount === 10) tenBlockersStarted.resolve()
        await blockersRelease.promise
      },
    })

    for (let index = 0; index < 10; index += 1) {
      instance.scheduleRetry('DELETED', server(`stop-blocker-${index}`))
    }
    await vi.advanceTimersByTimeAsync(5000)
    await tenBlockersStarted.promise
    instance.scheduleRetry('DELETED', queued)
    await vi.advanceTimersByTimeAsync(5000)

    instance.stop()
    blockersRelease.resolve()
    await flushMicrotasks()
    await flushMicrotasks()

    expect(enqueuedTargets).toEqual([])
  })

  it('resolves the current canonical server after periodic jitter', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0.5)
    const stale = server('web-search', 1)
    const current = server('web-search', 2)
    const state = new Map<string, McpServerCRD>([[stale.name, stale]])
    const reconciled: McpServerCRD[] = []
    const { instance } = coordinator({
      listServers: () => [stale],
      getCurrentServer: name => state.get(name),
      mutate: async (_type, selected) => {
        reconciled.push(selected)
      },
    })

    const resync = instance.runResync()
    await flushMicrotasks()
    state.set(current.name, current)
    await vi.advanceTimersByTimeAsync(2500)
    await resync

    expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
    instance.stop()
  })

  it('returns one global single-flight promise for overlapping resync requests', async () => {
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const selected = server('web-search')
    const release = deferred()
    const mutationStarted = deferred()
    const { instance, servers } = coordinator({
      mutate: async () => {
        mutationStarted.resolve()
        await release.promise
      },
    })
    servers.set(selected.name, selected)

    const first = instance.runResync()
    const overlapping = instance.runResync()
    expect(overlapping).toBe(first)
    await mutationStarted.promise

    release.resolve()
    await first
    instance.stop()
  })

  it('skips a resync key whose external-egress mutation is already in flight', async () => {
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const selected = server('web-search')
    const release = deferred()
    const mutationStarted = deferred()
    const mutate = vi.fn(async () => {
      mutationStarted.resolve()
      await release.promise
    })
    const { instance, servers } = coordinator({ mutate })
    servers.set(selected.name, selected)

    const active = instance.reconcile('MODIFIED', selected)
    await mutationStarted.promise
    await instance.runResync()

    expect(mutate).toHaveBeenCalledOnce()
    release.resolve()
    await active
    instance.stop()
  })

  it('resolves canonical state after entering the keyed resync queue', async () => {
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const stale = server('web-search', 1)
    const current = server('web-search', 2)
    const state = new Map<string, McpServerCRD>([[stale.name, stale]])
    const queueEntered = deferred()
    const releaseQueue = deferred()
    const reconciled: McpServerCRD[] = []
    const { instance } = coordinator({
      listServers: () => [stale],
      getCurrentServer: name => state.get(name),
      enqueue: async (_selected, work) => {
        queueEntered.resolve()
        await releaseQueue.promise
        await work()
      },
      mutate: async (_type, selected) => {
        reconciled.push(selected)
      },
    })

    const resync = instance.runResync()
    await queueEntered.promise
    state.set(current.name, current)
    releaseQueue.resolve()
    await resync

    expect(reconciled).toEqual([expect.objectContaining({ generation: 2 })])
    instance.stop()
  })

  it('keeps a startup gate false when desired revision changes during egress', async () => {
    const selected = server('web-search', 1)
    const current = server('web-search', 2)
    const state = new Map<string, McpServerCRD>([[selected.name, selected]])
    const mutationStarted = deferred()
    const releaseMutation = deferred()
    let selectedLease: (() => boolean) | undefined
    const { instance } = coordinator({
      getCurrentServer: name => state.get(name),
      mutate: async (_type, _server, options) => {
        selectedLease = options.isCurrent
        mutationStarted.resolve()
        await releaseMutation.promise
      },
    })

    const gates = instance.prepareStartupGates([selected])
    await mutationStarted.promise
    expect(selectedLease?.()).toBe(true)
    state.set(current.name, current)
    expect(selectedLease?.()).toBe(false)
    releaseMutation.resolve()

    await expect(gates.waitFor(selected)).resolves.toBe(false)
    instance.stop()
  })

  it('keeps a failed startup DNS gate isolated and recovers it through the dedicated retry', async () => {
    vi.useFakeTimers()
    const selected = server('dns-held-server')
    const dnsFailure = new Error('dns resolution failed')
    const mutate = vi.fn().mockRejectedValueOnce(dnsFailure).mockResolvedValue(undefined)
    const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    let instance!: ExternalEgressConvergenceCoordinator
    const built = coordinator({
      mutate,
      replay: async (type, current, retry) => {
        const completion = await instance.reconcile(type, current, { retry })
        completion?.complete()
      },
    })
    instance = built.instance
    built.servers.set(selected.name, selected)

    const gates = instance.prepareStartupGates([selected])
    await expect(gates.waitFor(selected)).resolves.toBe(false)
    expect(mutate).toHaveBeenCalledOnce()
    expect(error).toHaveBeenCalledWith(
      `[K8s] Initial external egress reconciliation failed for ${selected.namespace}/${selected.name}; ` +
        'runtime reconciliation will stay blocked until retry succeeds:',
      dnsFailure
    )

    await vi.advanceTimersByTimeAsync(5000)
    expect(mutate).toHaveBeenCalledTimes(2)
    await vi.advanceTimersByTimeAsync(30000)
    expect(mutate).toHaveBeenCalledTimes(2)
    instance.stop()
  })

  it('retires a direct mutation lease when desired revision changes in place', async () => {
    const selected = server('web-search', 1)
    const replacement = server('web-search', 2)
    const mutationStarted = deferred()
    const releaseMutation = deferred()
    let selectedLease: (() => boolean) | undefined
    const { instance, servers } = coordinator({
      mutate: async (_type, _server, options) => {
        selectedLease = options.isCurrent
        mutationStarted.resolve()
        await releaseMutation.promise
      },
    })
    servers.set(selected.name, selected)

    const mutation = instance.reconcile('MODIFIED', selected)
    await mutationStarted.promise
    expect(selectedLease?.()).toBe(true)

    servers.set(replacement.name, replacement)
    expect(selectedLease?.()).toBe(false)
    releaseMutation.resolve()
    await mutation
    instance.stop()
  })

  it('does not start a second mutation after stop releases an in-flight wait', async () => {
    const firstMutation = deferred()
    const selected = server('web-search')
    const mutate = vi.fn(async () => firstMutation.promise)
    const { instance, servers } = coordinator({ mutate })
    servers.set(selected.name, selected)

    const first = instance.reconcile('MODIFIED', selected)
    await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
    const waiting = instance.reconcile('MODIFIED', selected)
    instance.stop()
    firstMutation.resolve()
    await Promise.all([first, waiting])

    expect(mutate).toHaveBeenCalledOnce()
  })

  it('rechecks per-key in-flight ownership when multiple waiters resume together', async () => {
    const selected = server('web-search')
    const releases = [deferred(), deferred(), deferred()]
    const starts = [deferred(), deferred(), deferred()]
    let active = 0
    let maxActive = 0
    let call = 0
    const mutate = vi.fn(async () => {
      const index = call++
      active += 1
      maxActive = Math.max(maxActive, active)
      starts[index].resolve()
      await releases[index].promise
      active -= 1
    })
    const { instance, servers } = coordinator({ mutate })
    servers.set(selected.name, selected)

    const first = instance.reconcile('MODIFIED', selected)
    await starts[0].promise
    const second = instance.reconcile('MODIFIED', selected)
    const third = instance.reconcile('MODIFIED', selected)

    try {
      releases[0].resolve()
      await starts[1].promise
      await flushMicrotasks()
      expect(maxActive).toBe(1)
    } finally {
      for (const release of releases) release.resolve()
      await Promise.all([first, second, third])
      instance.stop()
    }
  })

  it('rechecks a caller lease after waiting for an in-flight mutation', async () => {
    const firstMutation = deferred()
    const selected = server('web-search')
    let current = true
    const mutate = vi.fn(async () => firstMutation.promise)
    const { instance, servers } = coordinator({ mutate })
    servers.set(selected.name, selected)

    const first = instance.reconcile('MODIFIED', selected)
    await vi.waitFor(() => expect(mutate).toHaveBeenCalledOnce())
    const waiting = instance.reconcile('MODIFIED', selected, {
      isCurrent: () => current,
    })
    current = false
    firstMutation.resolve()
    await Promise.all([first, waiting])

    expect(mutate).toHaveBeenCalledOnce()
    instance.stop()
  })

  it('keeps a retry pending while inventory authority is unavailable', async () => {
    vi.useFakeTimers()
    const selected = server('web-search')
    let authoritative = false
    const { instance, replay, servers } = coordinator({
      inventoryAuthoritative: () => authoritative,
    })
    servers.set(selected.name, selected)

    instance.scheduleRetry('ADDED', selected)
    await vi.advanceTimersByTimeAsync(5000)
    expect(replay).not.toHaveBeenCalled()

    authoritative = true
    await vi.advanceTimersByTimeAsync(14_999)
    expect(replay).not.toHaveBeenCalled()
    await vi.advanceTimersByTimeAsync(1)
    expect(replay).toHaveBeenCalledOnce()
    instance.stop()
  })

  it('skips periodic resync while inventory authority is unavailable', async () => {
    const selected = server('web-search')
    let authoritative = false
    const { instance, mutate, servers } = coordinator({
      inventoryAuthoritative: () => authoritative,
    })
    servers.set(selected.name, selected)

    await instance.runResync()
    expect(mutate).not.toHaveBeenCalled()

    authoritative = true
    await instance.runResync()
    expect(mutate).toHaveBeenCalledOnce()
    instance.stop()
  })

  it('does not resume periodic work from jitter after stop', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0.5)
    const selected = server('web-search')
    const { instance, mutate, servers } = coordinator()
    servers.set(selected.name, selected)

    const resync = instance.runResync()
    await flushMicrotasks()
    instance.stop()
    await vi.advanceTimersByTimeAsync(2500)
    await resync

    expect(mutate).not.toHaveBeenCalled()
  })

  it('does not start an eleventh startup mutation after stop releases the limiter', async () => {
    const release = deferred()
    const tenStarted = deferred()
    let active = 0
    const selected = Array.from({ length: 11 }, (_, index) => server(`startup-${index}`))
    const state = new Map(selected.map(item => [item.name, item]))
    const mutate = vi.fn(async () => {
      active += 1
      if (active === 10) tenStarted.resolve()
      await release.promise
    })
    const { instance } = coordinator({
      getCurrentServer: name => state.get(name),
      mutate,
    })

    const gates = instance.prepareStartupGates(selected)
    await tenStarted.promise
    instance.stop()
    release.resolve()
    await Promise.all(selected.map(item => gates.waitFor(item)))

    expect(mutate).toHaveBeenCalledTimes(10)
  })

  it('passes DELETE authority through to the watcher mutation without evaluating it', async () => {
    const selected = server('web-search')
    const deleteAllowed = vi.fn(async () => false)
    const mutate = vi.fn(async () => undefined)
    const { instance } = coordinator({ mutate })

    await instance.reconcile('DELETED', selected, { deleteAllowed })

    expect(mutate).toHaveBeenCalledWith('DELETED', selected, {
      deleteAllowed,
      isCurrent: expect.any(Function),
    })
    expect(deleteAllowed).not.toHaveBeenCalled()
    instance.stop()
  })

  it('owns periodic resync scheduling and stops it immediately', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const selected = server('web-search')
    const { instance, mutate, servers } = coordinator()
    servers.set(selected.name, selected)

    instance.startPeriodicResync(1, 1)
    await vi.advanceTimersByTimeAsync(1000)
    expect(mutate).toHaveBeenCalledOnce()

    instance.stop()
    await vi.advanceTimersByTimeAsync(5000)
    expect(mutate).toHaveBeenCalledOnce()
  })

  it('warns once when periodic resync is disabled (intervalSec <= 0) and arms no timer', async () => {
    vi.useFakeTimers()
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
    const selected = server('web-search')
    const { instance, mutate, servers } = coordinator()
    servers.set(selected.name, selected)

    instance.startPeriodicResync(0, 5)
    // No timer armed: advancing time never triggers a resync pass.
    await vi.advanceTimersByTimeAsync(60_000)
    expect(mutate).not.toHaveBeenCalled()

    expect(warnSpy).toHaveBeenCalledTimes(1)
    expect(warnSpy).toHaveBeenCalledWith(
      expect.stringContaining('External egress periodic DNS resync disabled')
    )

    // Called again while still disabled: no duplicate warning (once only).
    instance.startPeriodicResync(-1, 5)
    expect(warnSpy).toHaveBeenCalledTimes(1)

    instance.stop()
  })

  it('does not warn about a disabled resync when it is enabled (intervalSec > 0)', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
    const selected = server('web-search')
    const { instance, servers } = coordinator()
    servers.set(selected.name, selected)

    instance.startPeriodicResync(1, 1)
    expect(warnSpy).not.toHaveBeenCalledWith(
      expect.stringContaining('External egress periodic DNS resync disabled')
    )

    instance.stop()
  })

  it('arms one resync at a time, rescheduling only after each pass completes (H2)', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const selected = server('web-search')
    const release: Array<() => void> = []
    const mutate = vi.fn(
      () =>
        new Promise<void>(resolve => {
          release.push(resolve)
        })
    )
    const { instance, servers } = coordinator({ mutate })
    servers.set(selected.name, selected)

    instance.startPeriodicResync(10, 5)
    // First pass fires at the configured interval (10s) and then blocks.
    await vi.advanceTimersByTimeAsync(10_000)
    expect(mutate).toHaveBeenCalledOnce()

    // Complete the pass off the fixed grid (at t=17s). The successor is armed
    // relative to completion (t=27s), NOT on a fixed 20s/30s interval grid.
    await vi.advanceTimersByTimeAsync(7_000)
    release[0]()
    await flushMicrotasks()

    // A fixed setInterval would fire a second pass at t=20s; the self-
    // rescheduling timer must not, because it only re-armed at completion.
    await vi.advanceTimersByTimeAsync(3_000)
    expect(mutate).toHaveBeenCalledOnce()

    // The successor fires 10s after completion (t=27s).
    await vi.advanceTimersByTimeAsync(7_000)
    expect(mutate).toHaveBeenCalledTimes(2)

    instance.stop()
    release.forEach(resolve => resolve())
  })

  it('advances the next periodic resync to <= observed TTL/2 (H2, issue #299)', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const selected = server('web-search')
    // The mutation observes a low DNS TTL, ratcheting the coordinator's source
    // down; scheduleNext reads it AFTER the pass and must advance to <= TTL/2.
    let minObservedTtlMs = Infinity
    const mutate = vi.fn(async () => {
      minObservedTtlMs = 20_000
    })
    const { instance, servers } = coordinator({
      mutate,
      externalEgressRefreshMinTtlMs: () => minObservedTtlMs,
    })
    servers.set(selected.name, selected)

    // Interval 60s, floor 5s. No TTL observed yet, so the first pass fires at 60s.
    instance.startPeriodicResync(60, 5)
    await vi.advanceTimersByTimeAsync(59_999)
    expect(mutate).not.toHaveBeenCalled()
    await vi.advanceTimersByTimeAsync(1)
    expect(mutate).toHaveBeenCalledOnce()

    // TTL/2 = 10s now governs the next delay, far below the 60s backstop.
    await vi.advanceTimersByTimeAsync(9_999)
    expect(mutate).toHaveBeenCalledOnce()
    await vi.advanceTimersByTimeAsync(1)
    expect(mutate).toHaveBeenCalledTimes(2)

    instance.stop()
  })

  it('logs and absorbs a rejected periodic resync pass, then re-arms (no unhandledRejection)', async () => {
    vi.useFakeTimers()
    vi.spyOn(Math, 'random').mockReturnValue(0)
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    // Capture any process-level unhandledRejection: the pre-fix code left the
    // rejected `void runResync()` promise unhandled, which crashes modern Node.
    const unhandled: unknown[] = []
    const onUnhandled = (reason: unknown): void => {
      unhandled.push(reason)
    }
    process.on('unhandledRejection', onUnhandled)

    const selected = server('web-search')
    // Reject OUTSIDE the per-server try/catch in runResyncCore (that catch only
    // wraps the mutation). A rejecting `enqueue` propagates through the worker to
    // Promise.all -> runResyncCore -> runResync, so the whole pass rejects.
    const enqueue = vi.fn(async () => {
      throw new Error('resync enqueue boom')
    })
    const { instance, servers } = coordinator({ enqueue })
    servers.set(selected.name, selected)

    try {
      instance.startPeriodicResync(10, 5)
      // First pass fires at 10s and rejects.
      await vi.advanceTimersByTimeAsync(10_000)
      await flushMicrotasks()

      // The rejection is logged by the fix's .catch (RED without it: no such log).
      expect(errorSpy).toHaveBeenCalledWith(
        '[K8s] External egress periodic resync pass failed:',
        expect.any(Error)
      )
      expect(enqueue).toHaveBeenCalledTimes(1)

      // The loop RE-ARMS despite the failure: the successor fires 10s later.
      await vi.advanceTimersByTimeAsync(10_000)
      await flushMicrotasks()
      expect(enqueue).toHaveBeenCalledTimes(2)

      // The fix handles the rejection, so nothing escaped as unhandled.
      expect(unhandled).toEqual([])
    } finally {
      process.off('unhandledRejection', onUnhandled)
      instance.stop()
    }
  })
})