Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): make the DNS bounded-timeout fail-static (zach88 R1-B1/M1/M2/L1/L2)

Alfredo Lopez5c9be2ec875f
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index e1fece5c4..3fe048e88 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.224",
+ "version": "0.1.225",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.224",
+ "version": "0.1.225",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 08b70df35..e5807f1ab 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.224",
+ "version": "0.1.225",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostFleetScheduler.properties.test.ts b/host-context-controller/src/hostFleetScheduler.properties.test.ts
new file mode 100644
index 000000000..6f1ce7bb2
--- /dev/null
+++ b/host-context-controller/src/hostFleetScheduler.properties.test.ts
@@ -0,0 +1,114 @@
+// Property coverage for zach88 R1-M2 on PR #382: the Host fleet-scheduler
+// precedence functions gained an `inputRevision` dimension, and these are exactly
+// the (state, input) precedence functions whose algebraic invariants must hold as
+// the request shape evolves. No fast-check dependency — the behavioral domain is
+// small (24 request shapes), so total enumeration is exhaustive AND deterministic.
+import { describe, expect, it } from 'vitest'
+import {
+ type HostFleetReconcileMode,
+ type HostFleetReconcileRequest,
+ hostFleetRequestCovers,
+ mergeHostFleetRequests,
+} from './hostFleetScheduler'
+
+const MODES: HostFleetReconcileMode[] = ['full', 'lifecycle']
+const REVISIONS = [0, 1, 2, 3]
+const GENERATIONS: Array<number | undefined> = [undefined, 0, 1]
+
+const DOMAIN: HostFleetReconcileRequest[] = []
+for (const mode of MODES) {
+ for (const inputRevision of REVISIONS) {
+ for (const ccLifecycleGeneration of GENERATIONS) {
+ DOMAIN.push({
+ reason: `r-${mode}-${inputRevision}-${String(ccLifecycleGeneration)}`,
+ mode,
+ inputRevision,
+ ccLifecycleGeneration,
+ })
+ }
+ }
+}
+
+// The safety-relevant core, ignoring `reason`: merge deliberately takes the
+// requested reason, so `reason` is not part of the algebraic identity.
+function sameCore(a: HostFleetReconcileRequest, b: HostFleetReconcileRequest): boolean {
+ return (
+ a.mode === b.mode &&
+ a.inputRevision === b.inputRevision &&
+ a.ccLifecycleGeneration === b.ccLifecycleGeneration
+ )
+}
+
+// Expected undefined-aware max for the ccLifecycleGeneration lattice.
+function expectedGen(
+ a: HostFleetReconcileRequest,
+ b: HostFleetReconcileRequest
+): number | undefined {
+ if (a.ccLifecycleGeneration === undefined) return b.ccLifecycleGeneration
+ if (b.ccLifecycleGeneration === undefined) return a.ccLifecycleGeneration
+ return Math.max(a.ccLifecycleGeneration, b.ccLifecycleGeneration)
+}
+
+describe('hostFleetScheduler precedence functions — exhaustive property coverage', () => {
+ it('enumerates the full behavioral domain (24 request shapes)', () => {
+ expect(DOMAIN).toHaveLength(MODES.length * REVISIONS.length * GENERATIONS.length)
+ expect(DOMAIN).toHaveLength(24)
+ })
+
+ it('hostFleetRequestCovers is reflexive — every request covers itself', () => {
+ for (const a of DOMAIN) {
+ expect(hostFleetRequestCovers(a, a), `covers(${a.reason}, self)`).toBe(true)
+ }
+ })
+
+ it('mergeHostFleetRequests is idempotent on the safety core', () => {
+ for (const a of DOMAIN) {
+ expect(sameCore(mergeHostFleetRequests(a, a), a), `merge(a,a) core ≡ a for ${a.reason}`).toBe(
+ true
+ )
+ }
+ })
+
+ it('mergeHostFleetRequests is commutative on the safety core (reason aside)', () => {
+ for (const a of DOMAIN) {
+ for (const b of DOMAIN) {
+ expect(
+ sameCore(mergeHostFleetRequests(a, b), mergeHostFleetRequests(b, a)),
+ `merge commutes for ${a.reason} / ${b.reason}`
+ ).toBe(true)
+ }
+ }
+ })
+
+ it('mergeHostFleetRequests drops nothing — mode escalates to full, revision and generation take the max', () => {
+ for (const a of DOMAIN) {
+ for (const b of DOMAIN) {
+ const m = mergeHostFleetRequests(a, b)
+ expect(m.mode === 'full', `mode no-drop for ${a.reason} / ${b.reason}`).toBe(
+ a.mode === 'full' || b.mode === 'full'
+ )
+ expect(m.inputRevision, `revision no-drop for ${a.reason} / ${b.reason}`).toBe(
+ Math.max(a.inputRevision, b.inputRevision)
+ )
+ expect(m.ccLifecycleGeneration, `generation no-drop for ${a.reason} / ${b.reason}`).toBe(
+ expectedGen(a, b)
+ )
+ }
+ }
+ })
+
+ it('mergeHostFleetRequests is associative on the safety core', () => {
+ for (const a of DOMAIN) {
+ for (const b of DOMAIN) {
+ for (const c of DOMAIN) {
+ const left = mergeHostFleetRequests(mergeHostFleetRequests(a, b), c)
+ const right = mergeHostFleetRequests(a, mergeHostFleetRequests(b, c))
+ expect(
+ sameCore(left, right),
+ `merge associates for ${a.reason} / ${b.reason} / ${c.reason}`
+ ).toBe(true)
+ }
+ }
+ }
+ })
+})
diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.ts
index d872d6cad..2c44dc014 100644
--- a/host-context-controller/src/hostFleetScheduler.ts
+++ b/host-context-controller/src/hostFleetScheduler.ts
@@ -25,7 +25,10 @@ type HostFleetSchedulerDependencies = {
recordRequest: (result: HostFleetRequestResult) => void
}
-function hostFleetRequestCovers(
+// Exported for property coverage (hostFleetScheduler.properties.test.ts): this is
+// a (state, input) -> boolean precedence predicate whose invariants (reflexivity,
+// mode/revision/generation precedence) must hold as the request shape evolves.
+export function hostFleetRequestCovers(
active: HostFleetReconcileRequest,
requested: HostFleetReconcileRequest
): boolean {
@@ -36,7 +39,10 @@ function hostFleetRequestCovers(
return active.ccLifecycleGeneration === requested.ccLifecycleGeneration
}
-function mergeHostFleetRequests(
+// Exported for property coverage: this is the (state, input) -> state merge whose
+// idempotency / commutativity / associativity / no-drop invariants must survive
+// changes to the request shape (the PR added the `inputRevision` dimension).
+export function mergeHostFleetRequests(
pending: HostFleetReconcileRequest,
requested: HostFleetReconcileRequest
): HostFleetReconcileRequest {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 7a4000270..fb1ac6231 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -311,24 +311,26 @@ vi.mock('./hostReconciler', () => ({
},
}))
-vi.mock('./networkPolicyReconciler', () => ({
- sameContextDesiredRevision: (
- expected: { name: string; namespace: string; spec: unknown },
- current: { name: string; namespace: string; spec: unknown }
- ) =>
- expected.name === current.name &&
- expected.namespace === current.namespace &&
- JSON.stringify(expected.spec) === JSON.stringify(current.spec),
- NetworkPolicyReconciler: class {
- ensureDefaultPolicies = mocks.ensureDefaultPolicies
- fullReconcile = mocks.netPolFullReconcile
- reconcileExternalEgress = vi.fn()
- reconcileContext = vi.fn()
- reconcileDeleteContext = vi.fn()
- cleanupExternalEgress = vi.fn()
- hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory
- },
-}))
+vi.mock('./networkPolicyReconciler', async () => {
+ // Use the REAL safety-critical comparator (uid/generation-aware, canonicalized)
+ // instead of a hand-rolled stand-in that diverged from prod semantics; only the
+ // NetworkPolicyReconciler class is replaced with a test double.
+ const actual = await vi.importActual<typeof import('./networkPolicyReconciler')>(
+ './networkPolicyReconciler'
+ )
+ return {
+ sameContextDesiredRevision: actual.sameContextDesiredRevision,
+ NetworkPolicyReconciler: class {
+ ensureDefaultPolicies = mocks.ensureDefaultPolicies
+ fullReconcile = mocks.netPolFullReconcile
+ reconcileExternalEgress = vi.fn()
+ reconcileContext = vi.fn()
+ reconcileDeleteContext = vi.fn()
+ cleanupExternalEgress = vi.fn()
+ hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory
+ },
+ }
+})
vi.mock('./bindingPolicyReconciler', () => ({
BindingPolicyReconciler: class {
diff --git a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts
new file mode 100644
index 000000000..e3b56340b
--- /dev/null
+++ b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts
@@ -0,0 +1,200 @@
+// Regression for zach88 R1-B1 / R1-M1 on PR #382.
+// D4 fail-static: a bounded-timeout DNS failure is a TRANSIENT resolver error and
+// must FREEZE the accumulated external-egress set, even when the persisted window
+// has lapsed. Before the fix the synthetic timeout Error carried no `.code`, so
+// classifyDnsError mapped it to 'permanent', lapsed entries were pruned to zero,
+// and the reconcile failed and DELETED the egress policy — dropping the McpServer's
+// external egress. The existing timeout test (networkPolicyReconciler.test.ts) uses
+// a server with no accumulated policy, so it fails loud regardless of the
+// classification and cannot catch this; this seeds a lapsed accumulated policy so
+// the freeze-vs-prune decision is observable.
+import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
+import type { RecordWithTtl } from 'node:dns'
+import * as dns from 'node:dns/promises'
+import { NetworkPolicyReconciler } from './networkPolicyReconciler'
+import { McpServerCRD } from './types'
+
+vi.mock('./mcpServerSafety', async () => {
+ const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')
+ return {
+ ...actual,
+ confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),
+ }
+})
+
+const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))
+const resolve4Mock = vi.mocked(dns.resolve4) as unknown as Mock
+
+vi.mock('./config', () => ({
+ config: {
+ namespace: 'mcp-server',
+ controlPlaneNamespace: 'control-plane',
+ hostNamespace: 'mcp-host',
+ rpcProxyNamespace: 'rpc-proxy',
+ port: 8081,
+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],
+ k8sApiCidrs: [],
+ externalEgressDnsResolveTimeoutMs: 5_000,
+ externalEgressOverlapSec: 300,
+ externalEgressMaxEntries: 128,
+ },
+}))
+
+vi.mock('node:dns/promises', () => ({
+ resolve4: vi.fn().mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }]),
+}))
+
+vi.mock('./metrics', () => ({
+ networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },
+ networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },
+}))
+
+function makeMockNetworkingApi() {
+ return {
+ createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
+ listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
+ deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
+ }
+}
+
+function makeMockCustomApi() {
+ return {
+ getNamespacedCustomObject: vi
+ .fn()
+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),
+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({
+ metadata: { generation: 7, resourceVersion: 'status-rv-default' },
+ status: {},
+ }),
+ patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
+ }
+}
+
+function makeReconciler(
+ mockApi: ReturnType<typeof makeMockNetworkingApi>,
+ mockCustomApi: ReturnType<typeof makeMockCustomApi>
+): NetworkPolicyReconciler {
+ const kc = new k8s.KubeConfig()
+ kc.loadFromOptions({
+ clusters: [{ name: 'test', server: 'https://test' }],
+ users: [{ name: 'test' }],
+ contexts: [{ name: 'test', cluster: 'test', user: 'test' }],
+ currentContext: 'test',
+ })
+ const reconciler = new NetworkPolicyReconciler(kc, new Map())
+ const networkingApi = {
+ ...mockApi,
+ listNamespacedNetworkPolicy: async (args: { namespace: string }) => {
+ const response = await mockApi.listNamespacedNetworkPolicy(args)
+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {
+ const name = policy.metadata?.name ?? `unnamed-${index}`
+ return {
+ ...policy,
+ metadata: {
+ ...policy.metadata,
+ name,
+ namespace: policy.metadata?.namespace ?? args.namespace,
+ uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,
+ resourceVersion: policy.metadata?.resourceVersion ?? '1',
+ },
+ }
+ })
+ return { ...response, items }
+ },
+ }
+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi
+ ;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi
+ return reconciler
+}
+
+describe('R1-B1 regression: DNS timeout must fail-static (freeze), not prune', () => {
+ let mockApi: ReturnType<typeof makeMockNetworkingApi>
+ let mockCustomApi: ReturnType<typeof makeMockCustomApi>
+ let reconciler: NetworkPolicyReconciler
+
+ beforeEach(() => {
+ vi.useRealTimers()
+ mockApi = makeMockNetworkingApi()
+ mockCustomApi = makeMockCustomApi()
+ reconciler = makeReconciler(mockApi, mockCustomApi)
+ vi.clearAllMocks()
+ resolve4Mock.mockResolvedValue(rec('1.2.3.4'))
+ })
+
+ it('retains the accumulated /32 when the resolver stalls past the bounded timeout while the window has lapsed', async () => {
+ const server: McpServerCRD = {
+ name: 'frozen-mcp',
+ namespace: 'mcp-server',
+ spec: {
+ contextRef: 'dev',
+ image: 'frozen:latest',
+ transport: { type: 'streamableHttp', url: 'http://frozen:3000', port: 3000 },
+ egressBindings: [{ dns: 'api.example.com', port: 443 }],
+ },
+ }
+
+ // 1) Bootstrap: healthy resolve writes the policy with its STATE annotation.
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ const written = (
+ mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }
+ ).body
+ const annotations = { ...(written.metadata?.annotations ?? {}) }
+ expect(annotations['clerum.io/egress-fqdn-state']).toBeTruthy()
+
+ // 2) Lapse the persisted window: every entry expired in the past. This is the
+ // "sustained stall" precondition: TTL+overlap ran out while DNS was failing.
+ const state = JSON.parse(annotations['clerum.io/egress-fqdn-state']) as Array<{
+ expiresAt: number
+ }>
+ for (const e of state) e.expiresAt = Date.now() - 60_000
+ annotations['clerum.io/egress-fqdn-state'] = JSON.stringify(state)
+
+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({
+ items: [
+ {
+ metadata: {
+ name: written.metadata?.name,
+ labels: written.metadata?.labels,
+ annotations,
+ },
+ spec: written.spec,
+ },
+ ],
+ })
+ mockApi.createNamespacedNetworkPolicy.mockClear()
+ mockApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockApi.deleteNamespacedNetworkPolicy.mockClear()
+
+ // 3) Stall the resolver so the bounded-timeout wrapper fires (the synthetic
+ // timeout Error), and run the next reconcile under fake timers.
+ vi.useFakeTimers()
+ resolve4Mock.mockImplementationOnce(() => new Promise(() => undefined))
+ const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ const settled = reconcile.then(
+ () => ({ outcome: 'resolved' as const }),
+ (err: unknown) => ({ outcome: 'rejected' as const, err })
+ )
+ await vi.advanceTimersByTimeAsync(5_000)
+ const result = await settled
+
+ // 4) Fail-static (D4/H1): the timeout is a TRANSIENT resolver failure — the
+ // reconcile must freeze and keep serving the accumulated /32, not fail and
+ // not revoke the policy.
+ expect(result.outcome).toBe('resolved')
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ const writes = [
+ ...mockApi.createNamespacedNetworkPolicy.mock.calls,
+ ...mockApi.replaceNamespacedNetworkPolicy.mock.calls,
+ ] as Array<[{ body: k8s.V1NetworkPolicy }]>
+ for (const [call] of writes) {
+ const cidrs = (call.body.spec?.egress ?? []).flatMap(rule =>
+ (rule.to ?? []).map(t => t.ipBlock?.cidr)
+ )
+ expect(cidrs).toContain('1.2.3.4/32')
+ }
+ })
+})
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index 70eab2a1a..67acf8792 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -1738,9 +1738,17 @@ export class NetworkPolicyReconciler {
new Promise<never>((_, reject) => {
timeout = setTimeout(
() =>
+ // A bounded-timeout DNS failure is a TRANSIENT resolver error, not a
+ // permanent one. classifyDnsError keys on `.code`, and a codeless Error
+ // falls into the 'permanent' (NXDOMAIN/expire) branch, which prunes the
+ // accumulated egress instead of freezing it — breaking D4 fail-static.
+ // Tag it ETIMEDOUT (already in TRANSIENT_DNS_CODES) so it freezes.
reject(
- new Error(
- `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`
+ Object.assign(
+ new Error(
+ `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`
+ ),
+ { code: 'ETIMEDOUT' }
)
),
config.externalEgressDnsResolveTimeoutMs
diff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.ts
index 4bd14f5e6..dbdf29093 100644
--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts
+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts
@@ -151,8 +151,6 @@ function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>)
w.mcpServerCacheSynced = true
w.contextCacheSynced = true
w.hostCacheSynced = true
- w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration
- w.networkPolicyRevocationServerGeneration = w.mcpWatchGeneration
w.networkPolicyRevocationContextRevision = w.contextDesiredRevision
w.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision
}