Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(hcc): expose external-egress DNS retries-at-cap as a metric [R4-M4]

Alfredo Lopez79479660dead
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
index 176a62759..5fb6c7c41 100644
--- a/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.test.ts
@@ -4,6 +4,7 @@ import {
type ExternalEgressRetryHandle,
type ExternalEgressWatchEventType,
} from './externalEgressConvergenceCoordinator'
+import { externalEgressRetriesAtCap } from './metrics'
import type { McpServerCRD } from './types'
function deferred<T = void>() {
@@ -151,6 +152,41 @@ describe('ExternalEgressConvergenceCoordinator', () => {
instance.stop()
})
+ it('emits the retries-at-cap gauge when a retry reaches the capped backoff and clears it on success (R3-M4/R2-L1)', async () => {
+ vi.useFakeTimers()
+ const setSpy = vi.spyOn(externalEgressRetriesAtCap, 'set')
+ const selected = server('web-search')
+ let instance!: ExternalEgressConvergenceCoordinator
+ let succeed = false
+ instance = coordinator({
+ getCurrentServer: () => selected,
+ replay: async (type, current, retry) => {
+ if (succeed) {
+ retry.complete()
+ return
+ }
+ instance.scheduleRetry(type, current)
+ },
+ }).instance
+
+ instance.scheduleRetry('ADDED', selected)
+ // First fire re-arms attempt 2 (15s) — still below the cap: no gauge write.
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(setSpy).not.toHaveBeenCalled()
+
+ // Second fire re-arms attempt 3 (30s), the capped delay: one server at cap.
+ await vi.advanceTimersByTimeAsync(15000)
+ expect(setSpy).toHaveBeenLastCalledWith(1)
+
+ // Once the replay finally succeeds, the capped server is cleared back to 0.
+ succeed = true
+ await vi.advanceTimersByTimeAsync(30000)
+ expect(setSpy).toHaveBeenLastCalledWith(0)
+
+ instance.stop()
+ setSpy.mockRestore()
+ })
+
it('never clears a pending full replay after an egress-only success', async () => {
vi.useFakeTimers()
const selected = server('web-search')
diff --git a/host-context-controller/src/externalEgressConvergenceCoordinator.ts b/host-context-controller/src/externalEgressConvergenceCoordinator.ts
index 45b24d020..fe95ef295 100644
--- a/host-context-controller/src/externalEgressConvergenceCoordinator.ts
+++ b/host-context-controller/src/externalEgressConvergenceCoordinator.ts
@@ -1,3 +1,4 @@
+import { externalEgressRetriesAtCap } from './metrics'
import type { McpServerCRD } from './types'
const EXTERNAL_EGRESS_RETRY_DELAYS_MS = [5000, 15000, 30000] as const
@@ -131,6 +132,10 @@ export class ExternalEgressConvergenceCoordinator {
private retrySequence = 0
private readonly retryLimiter = new AsyncConcurrencyLimiter(EXTERNAL_EGRESS_MAX_CONCURRENCY)
private readonly retrySlots = new Map<string, object>()
+ // Keys whose retry backoff is currently pinned at the capped (maximum) delay.
+ // Its size drives the externalEgressRetriesAtCap gauge (R3-M4/R2-L1) so a
+ // binding stuck retrying forever is observable, not silently denied.
+ private readonly retriesAtCap = new Set<string>()
private readonly inFlight = new Map<string, Promise<void>>()
private resyncInFlight: Promise<void> | null = null
private resyncTimer: ReturnType<typeof setTimeout> | null = null
@@ -298,6 +303,8 @@ export class ExternalEgressConvergenceCoordinator {
const previousIntent = this.retryIntents.get(key)
if (previousIntent && !this.sameRetryTarget(previousIntent, retryType, retryServer)) {
this.retryAttempts.delete(key)
+ // A new target restarts the backoff ladder, so the key is no longer capped.
+ this.clearRetryAtCap(key)
}
const intent: ExternalEgressRetryIntent = {
@@ -328,6 +335,7 @@ export class ExternalEgressConvergenceCoordinator {
const delayMs = EXTERNAL_EGRESS_RETRY_DELAYS_MS[attempt - 1]
this.retryAttempts.set(key, attempt)
const capped = attempt === EXTERNAL_EGRESS_RETRY_DELAYS_MS.length
+ if (capped) this.markRetryAtCap(key)
console.warn(
`[K8s] Scheduling external egress retry ${attempt}/${EXTERNAL_EGRESS_RETRY_DELAYS_MS.length}` +
`${capped ? ' (capped)' : ''} ` +
@@ -412,6 +420,10 @@ export class ExternalEgressConvergenceCoordinator {
this.retryAttempts.clear()
this.retryIntents.clear()
this.retrySlots.clear()
+ if (this.retriesAtCap.size > 0) {
+ this.retriesAtCap.clear()
+ externalEgressRetriesAtCap.set(0)
+ }
this.inFlight.clear()
}
@@ -531,6 +543,18 @@ export class ExternalEgressConvergenceCoordinator {
}
this.retryAttempts.delete(key)
this.retryIntents.delete(key)
+ this.clearRetryAtCap(key)
+ }
+
+ private markRetryAtCap(key: string): void {
+ if (this.retriesAtCap.has(key)) return
+ this.retriesAtCap.add(key)
+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)
+ }
+
+ private clearRetryAtCap(key: string): void {
+ if (!this.retriesAtCap.delete(key)) return
+ externalEgressRetriesAtCap.set(this.retriesAtCap.size)
}
private sameRetryTarget(
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 08b57ca73..59465c3b3 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -225,3 +225,13 @@ export const hostDeleteCleanupTotal = counter({
help: 'Host deletion cleanup by outcome (queued/confirmed/completed/retried/superseded).',
labelNames: ['outcome'] as const,
})
+
+// External-egress DNS retry saturation (#205 audit R3-M4 / R2-L1). A binding
+// whose DNS never resolves is retried forever at the capped (maximum) backoff;
+// without this signal the still-denied binding converges nowhere yet stays
+// invisible. Tracks the COUNT of servers pinned at the cap — never a per-server
+// label — so cardinality stays flat regardless of fleet size.
+export const externalEgressRetriesAtCap = gauge({
+ name: 'clerum_hcc_external_egress_retries_at_cap',
+ help: 'McpServers whose external-egress DNS retry is pinned at the capped (maximum) backoff, i.e. repeatedly failing to converge.',
+})