Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
22.1 KB
Raw Blame
import { externalEgressRetriesAtCap } from './metrics'
import type { McpServerCRD } from './types'

const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const
const EXTERNAL_EGRESS_MAX_CONCURRENCY = 10
const EXTERNAL_EGRESS_RESYNC_JITTER_MS = 5000

export type ExternalEgressWatchEventType = 'ADDED' | 'MODIFIED' | 'DELETED'

export type ExternalEgressRetryHandle = {
  isCurrent: () => boolean
  complete: () => void
}

type ExternalEgressRetryIntent = {
  sequence: number
  type: ExternalEgressWatchEventType
  server: McpServerCRD
}

type ExternalEgressMutationOptions = {
  deleteAllowed?: () => Promise<boolean>
  isCurrent: () => boolean
}

type ExternalEgressReconcileOptions = {
  deleteAllowed?: () => Promise<boolean>
  retry?: ExternalEgressRetryHandle
  isCurrent?: () => boolean
}

type ExternalEgressCoordinatorDependencies = {
  listServers: () => McpServerCRD[]
  getCurrentServer: (name: string) => McpServerCRD | undefined
  inventoryAuthoritative: () => boolean
  sameDesiredRevision: (left: McpServerCRD, right: McpServerCRD) => boolean
  enqueue: (server: McpServerCRD, work: () => Promise<void>) => Promise<void>
  mutate: (
    type: ExternalEgressWatchEventType,
    server: McpServerCRD,
    options: ExternalEgressMutationOptions
  ) => Promise<void>
  replay: (
    type: ExternalEgressWatchEventType,
    server: McpServerCRD,
    retry: ExternalEgressRetryHandle
  ) => Promise<void>
  // Smallest DNS TTL (ms) observed across external-egress resolutions, or
  // Infinity if none. startPeriodicResync reads it to advance the next resync to
  // <= TTL/2 (H2, issue #299).
  externalEgressRefreshMinTtlMs: () => number
}

export type ExternalEgressStartupGates = {
  waitFor: (server: Pick<McpServerCRD, 'name' | 'namespace'>) => Promise<boolean>
}

class AsyncConcurrencyLimiter {
  private active = 0
  private readonly waiters: Array<(admitted: boolean) => void> = []
  private closed = false

  constructor(private readonly limit: number) {}

  async run<T>(work: () => Promise<T>): Promise<T | undefined> {
    if (this.closed) return undefined
    if (this.active >= this.limit) {
      const admitted = await new Promise<boolean>(resolve => this.waiters.push(resolve))
      if (!admitted) return undefined
    } else {
      this.active += 1
    }
    try {
      return await work()
    } finally {
      this.release()
    }
  }

  cancelPending(): void {
    if (this.closed) return
    this.closed = true
    for (const resume of this.waiters.splice(0)) {
      resume(false)
    }
  }

  private release(): void {
    this.active -= 1
    if (this.closed) return
    const resume = this.waiters.shift()
    if (!resume) return
    // Transfer the released permit before waking the waiter. Reserving it here
    // prevents a new caller from entering between wake-up and its microtask.
    this.active += 1
    resume(true)
  }
}

function delay(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms))
}

/**
 * External-egress resync delay (ms), H2 (issue #299). The configured interval is
 * the backstop; a finite observed DNS TTL advances the delay to <= TTL/2 so a
 * rotating low-TTL host is sampled every rotation. The floor is the hard lower
 * bound to avoid a hot-loop. Pure and unit-testable; consumed by
 * {@link ExternalEgressConvergenceCoordinator.startPeriodicResync}.
 */
export function externalEgressResyncDelayMs(
  intervalSec: number,
  floorSec: number,
  minObservedTtlMs = Infinity
): number {
  const floorMs = floorSec * 1000
  const configuredMs = intervalSec * 1000
  const ttlAwareMs = Number.isFinite(minObservedTtlMs)
    ? Math.min(configuredMs, minObservedTtlMs / 2)
    : configuredMs
  return Math.max(ttlAwareMs, floorMs)
}

/**
 * Owns temporal coordination for external-egress convergence. Kubernetes
 * authority and mutations remain behind callbacks supplied by the watcher.
 */
export class ExternalEgressConvergenceCoordinator {
  private readonly retryTimers = new Map<string, ReturnType<typeof setTimeout>>()
  private readonly retryAttempts = new Map<string, number>()
  private readonly retryIntents = new Map<string, ExternalEgressRetryIntent>()
  private retrySequence = 0
  private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
  private readonly retrySlots = new Map<string, object>()
  // Keys whose retry backoff is currently pinned at the capped (maximum) delay.
  // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a
  // binding stuck retrying forever is observable, not silently denied.
  private readonly retriesAtCap = new Set<string>()
  private readonly inFlight = new Map<string, Promise<void>>()
  private resyncInFlight: Promise<void> | null = null
  private resyncTimer: ReturnType<typeof setTimeout> | null = null
  private resyncDisabledWarned = false
  private stopped = false

  constructor(private readonly dependencies: ExternalEgressCoordinatorDependencies) {}

  startPeriodicResync(intervalSec: number, floorSec: number): void {
    if (this.stopped || this.resyncTimer) return
    if (intervalSec <= 0) {
      // Disabled. Warn once (mirrors the Host/SFS/GFS resync lanes) so the
      // operational consequence is visible: without periodic resync the
      // accumulated external-egress DNS allows are never refreshed, so a
      // binding whose upstream DNS rotates (issue #299) can drift until an
      // unrelated McpServer event triggers reconciliation.
      if (!this.resyncDisabledWarned) {
        this.resyncDisabledWarned = true
        console.warn(
          '[K8s] External egress periodic DNS resync disabled; accumulated DNS allows will not ' +
            'be refreshed, so a binding whose upstream DNS rotates (issue #299) can drift until ' +
            'another McpServer event triggers reconciliation.'
        )
      }
      return
    }
    // Self-rescheduling (not a fixed setInterval) so the delay can advance to
    // <= observed TTL/2 (H2, issue #299), and rescheduling AFTER each pass
    // completes guarantees no overlapping resyncs. runResync still dedupes any
    // in-flight pass; the floor is the hard lower bound against a hot-loop.
    const scheduleNext = (): void => {
      if (this.stopped) return
      const delayMs = externalEgressResyncDelayMs(
        intervalSec,
        floorSec,
        this.dependencies.externalEgressRefreshMinTtlMs()
      )
      this.resyncTimer = setTimeout(() => {
        // Re-arm FIRST (.finally) so a rejected pass still schedules its
        // successor — the "re-arms after completion" property must survive a
        // failure. Then .catch logs and absorbs the rejection so a rejecting
        // pass (or a throwing scheduleNext) never becomes an unhandledRejection,
        // which crashes the process on modern Node and would silently kill the
        // periodic resync loop (reopening the #299 DNS-drift window).
        void this.runResync()
          .finally(scheduleNext)
          .catch(error =>
            console.error('[K8s] External egress periodic resync pass failed:', error)
          )
      }, delayMs)
      this.resyncTimer.unref?.()
    }
    scheduleNext()
    console.log(
      `[K8s] External egress periodic DNS resync enabled (every ${intervalSec}s, advancing to <= TTL/2)`
    )
  }

  runResync(): Promise<void> {
    if (this.resyncInFlight) return this.resyncInFlight
    const run = this.runResyncCore()
    this.resyncInFlight = run
    const settle = () => {
      if (this.resyncInFlight === run) this.resyncInFlight = null
    }
    void run.then(settle, settle)
    return run
  }

  prepareStartupGates(
    servers: McpServerCRD[],
    isCurrent: () => boolean = this.dependencies.inventoryAuthoritative
  ): ExternalEgressStartupGates {
    const gates = new Map<string, Promise<boolean>>()
    const selectedServers = servers
      .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
      .sort((left, right) => this.key(left).localeCompare(this.key(right)))

    if (selectedServers.length > 0) {
      console.log(
        `[K8s] Reconciling external egress before startup runtime reconciliation for ` +
          `${selectedServers.length} McpServer(s)`
      )
    }

    // Startup admission is intentionally independent from retry and periodic
    // resync capacity, so a saturated recovery lane cannot head-of-line block
    // initial runtime gates for unrelated servers.
    const limiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
    for (const selected of selectedServers) {
      const key = this.key(selected)
      const gate = limiter
        .run(async () => {
          if (this.stopped || !isCurrent()) return false
          let ready = false
          await this.dependencies.enqueue(selected, async () => {
            if (this.stopped || !isCurrent()) return
            const current = this.dependencies.getCurrentServer(selected.name)
            if (
              !current ||
              current.namespace !== selected.namespace ||
              !this.dependencies.sameDesiredRevision(selected, current)
            ) {
              return
            }
            try {
              // Startup is an egress-only gate. Any pre-existing retry intent is
              // completed only by its full replay, never by this mutation.
              await this.reconcile('MODIFIED', current, { isCurrent })
              const latest = this.dependencies.getCurrentServer(selected.name)
              ready =
                !this.stopped &&
                isCurrent() &&
                latest !== undefined &&
                latest.namespace === selected.namespace &&
                this.dependencies.sameDesiredRevision(selected, latest)
            } catch (error) {
              console.error(
                `[K8s] Initial external egress reconciliation failed for ${key}; ` +
                  'runtime reconciliation will stay blocked until retry succeeds:',
                error
              )
              this.scheduleRetry('MODIFIED', current)
            }
          })
          return ready
        })
        .then(result => result ?? false)
      // Side-attach a logger so a rejection is never *unhandled* (which crashes
      // the process on modern Node) if fullReconcile aborts before waitFor
      // awaits this gate. This does NOT swallow the failure: gates.set stores
      // the original `gate`, so the awaited path in fullReconcile still rejects
      // and fails the pass loudly — this only covers the unawaited case.
      gate.catch(error => {
        console.error(
          `[K8s] External egress startup gate for ${key} rejected before it was awaited:`,
          error
        )
      })
      gates.set(key, gate)
    }

    return {
      waitFor: server => gates.get(this.key(server)) ?? Promise.resolve(true),
    }
  }

  async reconcile(
    type: ExternalEgressWatchEventType,
    server: McpServerCRD,
    options: ExternalEgressReconcileOptions = {}
  ): Promise<ExternalEgressRetryHandle | undefined> {
    if (!this.reconcileIsCurrent(type, server, options)) return undefined
    const key = this.key(server)
    let existing = this.inFlight.get(key)
    while (existing) {
      console.warn(`[K8s] Waiting for external egress reconcile for ${key}; already in flight`)
      await existing
      if (!this.reconcileIsCurrent(type, server, options)) return undefined
      existing = this.inFlight.get(key)
    }

    if (!this.reconcileIsCurrent(type, server, options)) return undefined
    const completion = options.retry ?? this.currentRetryHandle(key)
    const run = this.dependencies.mutate(type, server, {
      deleteAllowed: options.deleteAllowed,
      isCurrent: () => this.reconcileIsCurrent(type, server, options),
    })
    this.inFlight.set(key, run)
    try {
      await run
      return completion
    } finally {
      if (this.inFlight.get(key) === run) {
        this.inFlight.delete(key)
      }
    }
  }

  scheduleRetry(type: string, server: McpServerCRD): void {
    if (!this.isWatchEventType(type) || this.stopped) return

    const cached = this.dependencies.getCurrentServer(server.name)
    const retryType: ExternalEgressWatchEventType =
      cached && cached.namespace === server.namespace
        ? type === 'ADDED' && this.dependencies.sameDesiredRevision(server, cached)
          ? 'ADDED'
          : 'MODIFIED'
        : 'DELETED'
    const retryServer = cached && cached.namespace === server.namespace ? cached : server
    const key = this.key(server)
    const previousIntent = this.retryIntents.get(key)
    if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {
      this.retryAttempts.delete(key)
      // A new target restarts the backoff ladder, so the key is no longer capped.
      this.clearRetryAtCap(key)
    }

    const intent: ExternalEgressRetryIntent = {
      sequence: ++this.retrySequence,
      type: retryType,
      server: retryServer,
    }
    this.retryIntents.set(key, intent)
    // A timer or queued/active slot already owning the key consumes the newest
    // intent when it next enters the retry and per-server lanes.
    if (this.retryTimers.has(key) || this.retrySlots.has(key)) return
    this.armRetryTimer(key)
  }

  private armRetryTimer(key: string): void {
    if (
      this.stopped ||
      this.retryTimers.has(key) ||
      this.retrySlots.has(key) ||
      !this.retryIntents.has(key)
    ) {
      return
    }
    const attempt = Math.min(
      (this.retryAttempts.get(key) ?? 0) + 1,
      EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
    )
    const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
    this.retryAttempts.set(key, attempt)
    const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
    if (capped) this.markRetryAtCap(key)
    console.warn(
      `[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +
        `${capped ? ' (capped)' : ''} ` +
        `for McpServer "${this.retryIntents.get(key)!.server.name}" in ${delayMs}ms`
    )

    const timer = setTimeout(() => {
      if (this.retryTimers.get(key) !== timer) return
      this.retryTimers.delete(key)
      this.queueRetry(key)
    }, delayMs)
    this.retryTimers.set(key, timer)
  }

  private queueRetry(key: string): void {
    if (this.stopped || !this.retryIntents.has(key)) return
    if (this.retrySlots.has(key)) {
      // A timer can overlap work admitted through another lane. The durable
      // slot owns the latest intent and rearms it on exit if it remains live.
      return
    }

    const slot = {}
    this.retrySlots.set(key, slot)
    void this.retryLimiter
      .run(async () => {
        if (
          this.stopped ||
          this.retrySlots.get(key) !== slot ||
          !this.dependencies.inventoryAuthoritative()
        ) {
          return
        }
        // Do not capture intent before the limiter. Churn while capacity is
        // saturated is represented by one slot and resolved latest-wins here.
        const admittedIntent = this.retryIntents.get(key)
        if (!admittedIntent) return
        await this.dependencies.enqueue(admittedIntent.server, async () => {
          if (
            this.stopped ||
            this.retrySlots.get(key) !== slot ||
            !this.dependencies.inventoryAuthoritative()
          ) {
            return
          }
          // The keyed queue may also wait. Resolve the intent again so only the
          // latest desired revision or absence is replayed.
          const currentIntent = this.retryIntents.get(key)
          if (!currentIntent) return
          await this.replayIntent(currentIntent)
        })
      })
      .catch(error => {
        if (!this.stopped) {
          console.error(`[K8s] External egress retry execution failed for ${key}:`, error)
        }
      })
      .finally(() => {
        if (this.retrySlots.get(key) !== slot) return
        this.retrySlots.delete(key)
        // A replay may schedule its successor while still active, or a newer
        // intent may arrive after this slot selected work. Rearm only after the
        // key leaves the slot so one key never accumulates limiter backlog.
        if (!this.stopped && this.retryIntents.has(key) && !this.retryTimers.has(key)) {
          this.armRetryTimer(key)
        }
      })
  }

  stop(): void {
    if (this.stopped) return
    this.stopped = true
    this.retryLimiter.cancelPending()
    if (this.resyncTimer) {
      clearTimeout(this.resyncTimer)
      this.resyncTimer = null
    }
    for (const timer of this.retryTimers.values()) {
      clearTimeout(timer)
    }
    this.retryTimers.clear()
    this.retryAttempts.clear()
    this.retryIntents.clear()
    this.retrySlots.clear()
    if (this.retriesAtCap.size > 0) {
      this.retriesAtCap.clear()
      externalEgressRetriesAtCap.set(0)
    }
    this.inFlight.clear()
  }

  private async runResyncCore(): Promise<void> {
    if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
    const servers = this.dependencies
      .listServers()
      .filter(server => (server.spec.egressBindings?.length ?? 0) > 0)
      .sort((left, right) => this.key(left).localeCompare(this.key(right)))
    if (servers.length === 0) return

    console.log(`[K8s] Periodic external egress DNS resync: ${servers.length} McpServer(s)`)
    let index = 0
    const workers = Array.from(
      { length: Math.min(EXTERNAL_EGRESS_MAX_CONCURRENCY, servers.length) },
      async () => {
        while (!this.stopped && this.dependencies.inventoryAuthoritative()) {
          const server = servers[index++]
          if (!server) return
          const key = this.key(server)
          if (this.inFlight.has(key)) {
            console.log(`[K8s] Skipping external egress resync for ${key}; reconcile in flight`)
            continue
          }
          const jitterMs = Math.floor(Math.random() * EXTERNAL_EGRESS_RESYNC_JITTER_MS)
          if (jitterMs > 0) await delay(jitterMs)
          if (this.stopped || !this.dependencies.inventoryAuthoritative()) return

          await this.dependencies.enqueue(server, async () => {
            if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
            // The fleet snapshot selects work only. Desired state is resolved
            // after jitter and after entering the per-server FIFO lane.
            const current = this.dependencies.getCurrentServer(server.name)
            if (
              !current ||
              current.namespace !== server.namespace ||
              (current.spec.egressBindings?.length ?? 0) === 0
            ) {
              return
            }
            try {
              const retryIntent = this.retryIntents.get(key)
              if (retryIntent) {
                await this.replayIntent(retryIntent)
                return
              }
              // Periodic refresh is egress-only and therefore cannot complete
              // a retry intent that may appear while the mutation is running.
              await this.reconcile('MODIFIED', current, {
                isCurrent: this.dependencies.inventoryAuthoritative,
              })
            } catch (error) {
              console.error(`[K8s] External egress periodic resync failed for ${key}:`, error)
              this.scheduleRetry('MODIFIED', current)
            }
          })
        }
      }
    )
    await Promise.all(workers)
  }

  private async replayIntent(intent: ExternalEgressRetryIntent): Promise<void> {
    if (this.stopped || !this.dependencies.inventoryAuthoritative()) return
    const handle = this.retryHandle(intent)
    if (!handle.isCurrent()) return

    const current = this.dependencies.getCurrentServer(intent.server.name)
    const type: ExternalEgressWatchEventType =
      current && current.namespace === intent.server.namespace ? 'MODIFIED' : 'DELETED'
    const server = type === 'DELETED' ? intent.server : current!
    await this.dependencies.replay(type, server, handle)
  }

  private reconcileIsCurrent(
    type: ExternalEgressWatchEventType,
    server: McpServerCRD,
    options: ExternalEgressReconcileOptions
  ): boolean {
    const callerIsCurrent =
      !this.stopped &&
      (!options.retry || options.retry.isCurrent()) &&
      (!options.isCurrent || options.isCurrent())
    if (!callerIsCurrent || type === 'DELETED') return callerIsCurrent

    const current = this.dependencies.getCurrentServer(server.name)
    return (
      current !== undefined &&
      current.namespace === server.namespace &&
      this.dependencies.sameDesiredRevision(server, current)
    )
  }

  private currentRetryHandle(key: string): ExternalEgressRetryHandle | undefined {
    const intent = this.retryIntents.get(key)
    return intent ? this.retryHandle(intent) : undefined
  }

  private retryHandle(intent: ExternalEgressRetryIntent): ExternalEgressRetryHandle {
    return {
      isCurrent: () =>
        !this.stopped &&
        this.retryIntents.get(this.key(intent.server))?.sequence === intent.sequence,
      complete: () => {
        if (this.retryIntents.get(this.key(intent.server))?.sequence !== intent.sequence) return
        this.clearRetry(intent.server)
      },
    }
  }

  private clearRetry(server: Pick<McpServerCRD, 'name' | 'namespace'>): void {
    const key = this.key(server)
    const timer = this.retryTimers.get(key)
    if (timer) {
      clearTimeout(timer)
      this.retryTimers.delete(key)
    }
    this.retryAttempts.delete(key)
    this.retryIntents.delete(key)
    this.clearRetryAtCap(key)
  }

  private markRetryAtCap(key: string): void {
    if (this.retriesAtCap.has(key)) return
    this.retriesAtCap.add(key)
    externalEgressRetriesAtCap.set(this.retriesAtCap.size)
  }

  private clearRetryAtCap(key: string): void {
    if (!this.retriesAtCap.delete(key)) return
    externalEgressRetriesAtCap.set(this.retriesAtCap.size)
  }

  private sameRetryTarget(
    previous: ExternalEgressRetryIntent,
    type: ExternalEgressWatchEventType,
    server: McpServerCRD
  ): boolean {
    const previousPresent = previous.type !== 'DELETED'
    const nextPresent = type !== 'DELETED'
    return (
      previousPresent === nextPresent &&
      this.dependencies.sameDesiredRevision(previous.server, server)
    )
  }

  private key(server: Pick<McpServerCRD, 'name' | 'namespace'>): string {
    return `${server.namespace}/${server.name}`
  }

  private isWatchEventType(type: string): type is ExternalEgressWatchEventType {
    return type === 'ADDED' || type === 'MODIFIED' || type === 'DELETED'
  }
}