Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(hcc): expose external-egress DNS retries-at-cap as a metric [R4-M4]
Alfredo Lopez79479660dead
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.tsindex 176a62759..5fb6c7c41 100644--- a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts@@ -4,6 +4,7 @@ import {type ExternalEgressRetryHandle,type ExternalEgressWatchEventType,} from './externalEgressConvergenceCoordinator'+import { externalEgressRetriesAtCap } from './metrics'import type { McpServerCRD } from './types'function deferred<T = void>() {@@ -151,6 +152,41 @@ describe('ExternalEgressConvergenceCoordinator', () => {instance.stop()})+ it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {+ vi.useFakeTimers()+ const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')+ const selected = server('web-search')+ let instance!: ExternalEgressConvergenceCoordinator+ let succeed = false+ instance = coordinator({+ getCurrentServer: () => selected,+ replay: async (type, current, retry) => {+ if (succeed) {+ retry.complete()+ return+ }+ instance.scheduleRetry(type, current)+ },+ }).instance++ instance.scheduleRetry('ADDED', selected)+ // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.+ await vi.advanceTimersByTimeAsync(5000)+ expect(setSpy).not.toHaveBeenCalled()++ // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.+ await vi.advanceTimersByTimeAsync(15000)+ expect(setSpy).toHaveBeenLastCalledWith(1)++ // Once the replay finally succeeds, the capped server is cleared back to 0.+ succeed = true+ await vi.advanceTimersByTimeAsync(30000)+ expect(setSpy).toHaveBeenLastCalledWith(0)++ instance.stop()+ setSpy.mockRestore()+ })+it('never clears a pending full replay after an egress-only success', async () => {vi.useFakeTimers()const selected = server('web-search')diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.tsindex 45b24d020..fe95ef295 100644--- a/host-context-controller/src/externalEgressConvergenceCoordinator.ts+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.ts@@ -1,3 +1,4 @@+import { externalEgressRetriesAtCap } from './metrics'import type { McpServerCRD } from './types'const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const@@ -131,6 +132,10 @@ export class ExternalEgressConvergenceCoordinator {private retrySequence = 0private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)private readonly retrySlots = new Map<string, object>()+ // Keys whose retry backoff is currently pinned at the capped (maximum) delay.+ // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a+ // binding stuck retrying forever is observable, not silently denied.+ private readonly retriesAtCap = new Set<string>()private readonly inFlight = new Map<string, Promise<void>>()private resyncInFlight: Promise<void> | null = nullprivate resyncTimer: ReturnType<typeof setTimeout> | null = null@@ -298,6 +303,8 @@ export class ExternalEgressConvergenceCoordinator {const previousIntent = this.retryIntents.get(key)if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {this.retryAttempts.delete(key)+ // A new target restarts the backoff ladder, so the key is no longer capped.+ this.clearRetryAtCap(key)}const intent: ExternalEgressRetryIntent = {@@ -328,6 +335,7 @@ export class ExternalEgressConvergenceCoordinator {const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]this.retryAttempts.set(key, attempt)const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length+ if (capped) this.markRetryAtCap(key)console.warn(`[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +`${capped ? ' (capped)' : ''} ` +@@ -412,6 +420,10 @@ export class ExternalEgressConvergenceCoordinator {this.retryAttempts.clear()this.retryIntents.clear()this.retrySlots.clear()+ if (this.retriesAtCap.size > 0) {+ this.retriesAtCap.clear()+ externalEgressRetriesAtCap.set(0)+ }this.inFlight.clear()}@@ -531,6 +543,18 @@ export class ExternalEgressConvergenceCoordinator {}this.retryAttempts.delete(key)this.retryIntents.delete(key)+ this.clearRetryAtCap(key)+ }++ private markRetryAtCap(key: string): void {+ if (this.retriesAtCap.has(key)) return+ this.retriesAtCap.add(key)+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)+ }++ private clearRetryAtCap(key: string): void {+ if (!this.retriesAtCap.delete(key)) return+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)}private sameRetryTarget(diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 08b57ca73..59465c3b3 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -225,3 +225,13 @@ export const hostDeleteCleanupTotal = counter({help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',labelNames: ['outcome'] as const,})++// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding+// whose DNS never resolves is retried forever at the capped (maximum) backoff;+// without this signal the still-denied binding converges nowhere yet stays+// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server+// label — so cardinality stays flat regardless of fleet size.+export const externalEgressRetriesAtCap = gauge({+ name: 'clerum_hcc_external_egress_retries_at_cap',+ help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',+})