Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): make the DNS bounded-timeout fail-static (zach88 R1-B1/M1/M2/L1/L2)
Alfredo Lopez5c9be2ec875f
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex e1fece5c4..3fe048e88 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.224",+ "version": "0.1.225","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.224",+ "version": "0.1.225","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 08b70df35..e5807f1ab 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.224",+ "version": "0.1.225","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostFleetScheduler.properties.test.ts b/host-context-controller/src/hostFleetScheduler.properties.test.tsnew file mode 100644index 000000000..6f1ce7bb2--- /dev/null+++ b/host-context-controller/src/hostFleetScheduler.properties.test.ts@@ -0,0 +1,114 @@+// Property coverage for zach88 R1-M2 on PR #382: the Host fleet-scheduler+// precedence functions gained an `inputRevision` dimension, and these are exactly+// the (state, input) precedence functions whose algebraic invariants must hold as+// the request shape evolves. No fast-check dependency — the behavioral domain is+// small (24 request shapes), so total enumeration is exhaustive AND deterministic.+import { describe, expect, it } from 'vitest'+import {+ type HostFleetReconcileMode,+ type HostFleetReconcileRequest,+ hostFleetRequestCovers,+ mergeHostFleetRequests,+} from './hostFleetScheduler'++const MODES: HostFleetReconcileMode[] = ['full', 'lifecycle']+const REVISIONS = [0, 1, 2, 3]+const GENERATIONS: Array<number | undefined> = [undefined, 0, 1]++const DOMAIN: HostFleetReconcileRequest[] = []+for (const mode of MODES) {+ for (const inputRevision of REVISIONS) {+ for (const ccLifecycleGeneration of GENERATIONS) {+ DOMAIN.push({+ reason: `r-${mode}-${inputRevision}-${String(ccLifecycleGeneration)}`,+ mode,+ inputRevision,+ ccLifecycleGeneration,+ })+ }+ }+}++// The safety-relevant core, ignoring `reason`: merge deliberately takes the+// requested reason, so `reason` is not part of the algebraic identity.+function sameCore(a: HostFleetReconcileRequest, b: HostFleetReconcileRequest): boolean {+ return (+ a.mode === b.mode &&+ a.inputRevision === b.inputRevision &&+ a.ccLifecycleGeneration === b.ccLifecycleGeneration+ )+}++// Expected undefined-aware max for the ccLifecycleGeneration lattice.+function expectedGen(+ a: HostFleetReconcileRequest,+ b: HostFleetReconcileRequest+): number | undefined {+ if (a.ccLifecycleGeneration === undefined) return b.ccLifecycleGeneration+ if (b.ccLifecycleGeneration === undefined) return a.ccLifecycleGeneration+ return Math.max(a.ccLifecycleGeneration, b.ccLifecycleGeneration)+}++describe('hostFleetScheduler precedence functions — exhaustive property coverage', () => {+ it('enumerates the full behavioral domain (24 request shapes)', () => {+ expect(DOMAIN).toHaveLength(MODES.length * REVISIONS.length * GENERATIONS.length)+ expect(DOMAIN).toHaveLength(24)+ })++ it('hostFleetRequestCovers is reflexive — every request covers itself', () => {+ for (const a of DOMAIN) {+ expect(hostFleetRequestCovers(a, a), `covers(${a.reason}, self)`).toBe(true)+ }+ })++ it('mergeHostFleetRequests is idempotent on the safety core', () => {+ for (const a of DOMAIN) {+ expect(sameCore(mergeHostFleetRequests(a, a), a), `merge(a,a) core ≡ a for ${a.reason}`).toBe(+ true+ )+ }+ })++ it('mergeHostFleetRequests is commutative on the safety core (reason aside)', () => {+ for (const a of DOMAIN) {+ for (const b of DOMAIN) {+ expect(+ sameCore(mergeHostFleetRequests(a, b), mergeHostFleetRequests(b, a)),+ `merge commutes for ${a.reason} / ${b.reason}`+ ).toBe(true)+ }+ }+ })++ it('mergeHostFleetRequests drops nothing — mode escalates to full, revision and generation take the max', () => {+ for (const a of DOMAIN) {+ for (const b of DOMAIN) {+ const m = mergeHostFleetRequests(a, b)+ expect(m.mode === 'full', `mode no-drop for ${a.reason} / ${b.reason}`).toBe(+ a.mode === 'full' || b.mode === 'full'+ )+ expect(m.inputRevision, `revision no-drop for ${a.reason} / ${b.reason}`).toBe(+ Math.max(a.inputRevision, b.inputRevision)+ )+ expect(m.ccLifecycleGeneration, `generation no-drop for ${a.reason} / ${b.reason}`).toBe(+ expectedGen(a, b)+ )+ }+ }+ })++ it('mergeHostFleetRequests is associative on the safety core', () => {+ for (const a of DOMAIN) {+ for (const b of DOMAIN) {+ for (const c of DOMAIN) {+ const left = mergeHostFleetRequests(mergeHostFleetRequests(a, b), c)+ const right = mergeHostFleetRequests(a, mergeHostFleetRequests(b, c))+ expect(+ sameCore(left, right),+ `merge associates for ${a.reason} / ${b.reason} / ${c.reason}`+ ).toBe(true)+ }+ }+ }+ })+})diff --git a/host-context-controller/src/hostFleetScheduler.ts b/host-context-controller/src/hostFleetScheduler.tsindex d872d6cad..2c44dc014 100644--- a/host-context-controller/src/hostFleetScheduler.ts+++ b/host-context-controller/src/hostFleetScheduler.ts@@ -25,7 +25,10 @@ type HostFleetSchedulerDependencies = {recordRequest: (result: HostFleetRequestResult) => void}-function hostFleetRequestCovers(+// Exported for property coverage (hostFleetScheduler.properties.test.ts): this is+// a (state, input) -> boolean precedence predicate whose invariants (reflexivity,+// mode/revision/generation precedence) must hold as the request shape evolves.+export function hostFleetRequestCovers(active: HostFleetReconcileRequest,requested: HostFleetReconcileRequest): boolean {@@ -36,7 +39,10 @@ function hostFleetRequestCovers(return active.ccLifecycleGeneration === requested.ccLifecycleGeneration}-function mergeHostFleetRequests(+// Exported for property coverage: this is the (state, input) -> state merge whose+// idempotency / commutativity / associativity / no-drop invariants must survive+// changes to the request shape (the PR added the `inputRevision` dimension).+export function mergeHostFleetRequests(pending: HostFleetReconcileRequest,requested: HostFleetReconcileRequest): HostFleetReconcileRequest {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 7a4000270..fb1ac6231 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -311,24 +311,26 @@ vi.mock('./hostReconciler', () => ({},}))-vi.mock('./networkPolicyReconciler', () => ({- sameContextDesiredRevision: (- expected: { name: string; namespace: string; spec: unknown },- current: { name: string; namespace: string; spec: unknown }- ) =>- expected.name === current.name &&- expected.namespace === current.namespace &&- JSON.stringify(expected.spec) === JSON.stringify(current.spec),- NetworkPolicyReconciler: class {- ensureDefaultPolicies = mocks.ensureDefaultPolicies- fullReconcile = mocks.netPolFullReconcile- reconcileExternalEgress = vi.fn()- reconcileContext = vi.fn()- reconcileDeleteContext = vi.fn()- cleanupExternalEgress = vi.fn()- hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory- },-}))+vi.mock('./networkPolicyReconciler', async () => {+ // Use the REAL safety-critical comparator (uid/generation-aware, canonicalized)+ // instead of a hand-rolled stand-in that diverged from prod semantics; only the+ // NetworkPolicyReconciler class is replaced with a test double.+ const actual = await vi.importActual<typeof import('./networkPolicyReconciler')>(+ './networkPolicyReconciler'+ )+ return {+ sameContextDesiredRevision: actual.sameContextDesiredRevision,+ NetworkPolicyReconciler: class {+ ensureDefaultPolicies = mocks.ensureDefaultPolicies+ fullReconcile = mocks.netPolFullReconcile+ reconcileExternalEgress = vi.fn()+ reconcileContext = vi.fn()+ reconcileDeleteContext = vi.fn()+ cleanupExternalEgress = vi.fn()+ hasCertifiedSafetyInventory = mocks.hasCertifiedSafetyInventory+ },+ }+})vi.mock('./bindingPolicyReconciler', () => ({BindingPolicyReconciler: class {diff --git a/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.tsnew file mode 100644index 000000000..e3b56340b--- /dev/null+++ b/host-context-controller/src/networkPolicyReconciler.egressFailStatic.test.ts@@ -0,0 +1,200 @@+// Regression for zach88 R1-B1 / R1-M1 on PR #382.+// D4 fail-static: a bounded-timeout DNS failure is a TRANSIENT resolver error and+// must FREEZE the accumulated external-egress set, even when the persisted window+// has lapsed. Before the fix the synthetic timeout Error carried no `.code`, so+// classifyDnsError mapped it to 'permanent', lapsed entries were pruned to zero,+// and the reconcile failed and DELETED the egress policy — dropping the McpServer's+// external egress. The existing timeout test (networkPolicyReconciler.test.ts) uses+// a server with no accumulated policy, so it fails loud regardless of the+// classification and cannot catch this; this seeds a lapsed accumulated policy so+// the freeze-vs-prune decision is observable.+import { type Mock, beforeEach, describe, expect, it, vi } from 'vitest'+import * as k8s from '@kubernetes/client-node'+import type { RecordWithTtl } from 'node:dns'+import * as dns from 'node:dns/promises'+import { NetworkPolicyReconciler } from './networkPolicyReconciler'+import { McpServerCRD } from './types'++vi.mock('./mcpServerSafety', async () => {+ const actual = await vi.importActual<typeof import('./mcpServerSafety')>('./mcpServerSafety')+ return {+ ...actual,+ confirmAuthoritativeMcpServerAbsence: vi.fn(actual.confirmAuthoritativeMcpServerAbsence),+ }+})++const rec = (...ips: string[]): RecordWithTtl[] => ips.map(address => ({ address, ttl: 300 }))+const resolve4Mock = vi.mocked(dns.resolve4) as unknown as Mock++vi.mock('./config', () => ({+ config: {+ namespace: 'mcp-server',+ controlPlaneNamespace: 'control-plane',+ hostNamespace: 'mcp-host',+ rpcProxyNamespace: 'rpc-proxy',+ port: 8081,+ runtimeNamespaces: ['mcp-server', 'mcp-host', 'sandbox-recipes', 'rpc-proxy'],+ k8sApiCidrs: [],+ externalEgressDnsResolveTimeoutMs: 5_000,+ externalEgressOverlapSec: 300,+ externalEgressMaxEntries: 128,+ },+}))++vi.mock('node:dns/promises', () => ({+ resolve4: vi.fn().mockResolvedValue([{ address: '1.2.3.4', ttl: 300 }]),+}))++vi.mock('./metrics', () => ({+ networkPolicySafetyPassDurationSeconds: { observe: vi.fn() },+ networkPolicySafetyPassPoliciesTotal: { inc: vi.fn() },+}))++function makeMockNetworkingApi() {+ return {+ createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),+ readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),+ listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),+ deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),+ }+}++function makeMockCustomApi() {+ return {+ getNamespacedCustomObject: vi+ .fn()+ .mockRejectedValue(Object.assign(new Error('not found'), { code: 404 })),+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),+ getNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({+ metadata: { generation: 7, resourceVersion: 'status-rv-default' },+ status: {},+ }),+ patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),+ }+}++function makeReconciler(+ mockApi: ReturnType<typeof makeMockNetworkingApi>,+ mockCustomApi: ReturnType<typeof makeMockCustomApi>+): NetworkPolicyReconciler {+ const kc = new k8s.KubeConfig()+ kc.loadFromOptions({+ clusters: [{ name: 'test', server: 'https://test' }],+ users: [{ name: 'test' }],+ contexts: [{ name: 'test', cluster: 'test', user: 'test' }],+ currentContext: 'test',+ })+ const reconciler = new NetworkPolicyReconciler(kc, new Map())+ const networkingApi = {+ ...mockApi,+ listNamespacedNetworkPolicy: async (args: { namespace: string }) => {+ const response = await mockApi.listNamespacedNetworkPolicy(args)+ const items = (response.items ?? []).map((policy: k8s.V1NetworkPolicy, index: number) => {+ const name = policy.metadata?.name ?? `unnamed-${index}`+ return {+ ...policy,+ metadata: {+ ...policy.metadata,+ name,+ namespace: policy.metadata?.namespace ?? args.namespace,+ uid: policy.metadata?.uid ?? `${args.namespace}:${name}:uid`,+ resourceVersion: policy.metadata?.resourceVersion ?? '1',+ },+ }+ })+ return { ...response, items }+ },+ }+ ;(reconciler as unknown as { networkingApi: unknown }).networkingApi = networkingApi+ ;(reconciler as unknown as { customApi: unknown }).customApi = mockCustomApi+ return reconciler+}++describe('R1-B1 regression: DNS timeout must fail-static (freeze), not prune', () => {+ let mockApi: ReturnType<typeof makeMockNetworkingApi>+ let mockCustomApi: ReturnType<typeof makeMockCustomApi>+ let reconciler: NetworkPolicyReconciler++ beforeEach(() => {+ vi.useRealTimers()+ mockApi = makeMockNetworkingApi()+ mockCustomApi = makeMockCustomApi()+ reconciler = makeReconciler(mockApi, mockCustomApi)+ vi.clearAllMocks()+ resolve4Mock.mockResolvedValue(rec('1.2.3.4'))+ })++ it('retains the accumulated /32 when the resolver stalls past the bounded timeout while the window has lapsed', async () => {+ const server: McpServerCRD = {+ name: 'frozen-mcp',+ namespace: 'mcp-server',+ spec: {+ contextRef: 'dev',+ image: 'frozen:latest',+ transport: { type: 'streamableHttp', url: 'http://frozen:3000', port: 3000 },+ egressBindings: [{ dns: 'api.example.com', port: 443 }],+ },+ }++ // 1) Bootstrap: healthy resolve writes the policy with its STATE annotation.+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ const written = (+ mockApi.createNamespacedNetworkPolicy.mock.calls[0][0] as { body: k8s.V1NetworkPolicy }+ ).body+ const annotations = { ...(written.metadata?.annotations ?? {}) }+ expect(annotations['clerum.io/egress-fqdn-state']).toBeTruthy()++ // 2) Lapse the persisted window: every entry expired in the past. This is the+ // "sustained stall" precondition: TTL+overlap ran out while DNS was failing.+ const state = JSON.parse(annotations['clerum.io/egress-fqdn-state']) as Array<{+ expiresAt: number+ }>+ for (const e of state) e.expiresAt = Date.now() - 60_000+ annotations['clerum.io/egress-fqdn-state'] = JSON.stringify(state)++ mockApi.listNamespacedNetworkPolicy.mockResolvedValue({+ items: [+ {+ metadata: {+ name: written.metadata?.name,+ labels: written.metadata?.labels,+ annotations,+ },+ spec: written.spec,+ },+ ],+ })+ mockApi.createNamespacedNetworkPolicy.mockClear()+ mockApi.replaceNamespacedNetworkPolicy.mockClear()+ mockApi.deleteNamespacedNetworkPolicy.mockClear()++ // 3) Stall the resolver so the bounded-timeout wrapper fires (the synthetic+ // timeout Error), and run the next reconcile under fake timers.+ vi.useFakeTimers()+ resolve4Mock.mockImplementationOnce(() => new Promise(() => undefined))+ const reconcile = reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ const settled = reconcile.then(+ () => ({ outcome: 'resolved' as const }),+ (err: unknown) => ({ outcome: 'rejected' as const, err })+ )+ await vi.advanceTimersByTimeAsync(5_000)+ const result = await settled++ // 4) Fail-static (D4/H1): the timeout is a TRANSIENT resolver failure — the+ // reconcile must freeze and keep serving the accumulated /32, not fail and+ // not revoke the policy.+ expect(result.outcome).toBe('resolved')+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ const writes = [+ ...mockApi.createNamespacedNetworkPolicy.mock.calls,+ ...mockApi.replaceNamespacedNetworkPolicy.mock.calls,+ ] as Array<[{ body: k8s.V1NetworkPolicy }]>+ for (const [call] of writes) {+ const cidrs = (call.body.spec?.egress ?? []).flatMap(rule =>+ (rule.to ?? []).map(t => t.ipBlock?.cidr)+ )+ expect(cidrs).toContain('1.2.3.4/32')+ }+ })+})diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 70eab2a1a..67acf8792 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -1738,9 +1738,17 @@ export class NetworkPolicyReconciler {new Promise<never>((_, reject) => {timeout = setTimeout(() =>+ // A bounded-timeout DNS failure is a TRANSIENT resolver error, not a+ // permanent one. classifyDnsError keys on `.code`, and a codeless Error+ // falls into the 'permanent' (NXDOMAIN/expire) branch, which prunes the+ // accumulated egress instead of freezing it — breaking D4 fail-static.+ // Tag it ETIMEDOUT (already in TRANSIENT_DNS_CODES) so it freezes.reject(- new Error(- `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`+ Object.assign(+ new Error(+ `DNS resolution timed out after ${config.externalEgressDnsResolveTimeoutMs}ms`+ ),+ { code: 'ETIMEDOUT' })),config.externalEgressDnsResolveTimeoutMsdiff --git a/host-context-controller/src/readinessSafetyFence.integration.test.ts b/host-context-controller/src/readinessSafetyFence.integration.test.tsindex 4bd14f5e6..dbdf29093 100644--- a/host-context-controller/src/readinessSafetyFence.integration.test.ts+++ b/host-context-controller/src/readinessSafetyFence.integration.test.ts@@ -151,8 +151,6 @@ function alignRevocationCounters(watcher: InstanceType<typeof McpServerWatcher>)w.mcpServerCacheSynced = truew.contextCacheSynced = truew.hostCacheSynced = true- w.networkPolicyRevocationContextGeneration = w.contextWatchGeneration- w.networkPolicyRevocationServerGeneration = w.mcpWatchGenerationw.networkPolicyRevocationContextRevision = w.contextDesiredRevisionw.networkPolicyRevocationServerRevision = w.mcpServerDesiredRevision}