Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(egress): fail-static on a fault with a live policy, and stop dropping rejections
Alfredo Lopez043593ede1c3
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 6704209ff..3929e0371 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.288",+ "version": "0.1.289","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.288",+ "version": "0.1.289","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex ac6bbd4bb..2b103109a 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.288",+ "version": "0.1.289","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.tsindex 5e40184da..f182dfb3e 100644--- a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts@@ -223,9 +223,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditconst warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)injectPostResolutionTypeError()- await expect(- reconciler.reconcileExternalEgress(server, { isCurrent: () => true })- ).rejects.toThrow(TypeError)+ // No throw: there is a live policy, so the fault is served fail-static (T7).+ // Everything else this test pins is unchanged.+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })// The operator must not be pointed at DNS for a fault in the reconciler.expect(warn.mock.calls.flat().join('\n')).not.toContain('permanent DNS failure')@@ -247,9 +247,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditawait seedAccumulatedPolicy({ lapsed: true })injectPostResolutionTypeError()- await expect(- reconciler.reconcileExternalEgress(server, { isCurrent: () => true })- ).rejects.toThrow(TypeError)+ // No throw: the policy is live (its window lapsed, the object did not), so+ // the fault is served fail-static (T7). The prune assertion is the point.+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })// Pre-fix this prunes the lapsed set to zero, hits the `continue`, and the// policy is swept as undesired. D4 fail-static must hold for a fault that is@@ -310,9 +310,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit// The operator removes a.example.com; b.example.com faults after resolution.injectPostResolutionTypeError()- await expect(- reconciler.reconcileExternalEgress(bindings(['b.example.com']), { isCurrent: () => true })- ).rejects.toThrow(TypeError)+ // No throw: b.example.com has a live policy, so the fault is served+ // fail-static (T7). What this test pins is that the cleanup still ran.+ await reconciler.reconcileExternalEgress(bindings(['b.example.com']), {+ isCurrent: () => true,+ })// Exactly one delete, and it is A's. `toHaveBeenCalledWith` alone is a// membership check, not an exclusivity one: it stays green when the cleanup@@ -328,6 +330,73 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit)})+ it('T7 serves the frozen set and does NOT block the workload when a fault has a live policy', async () => {+ // Review of #567, major 3. The throw reaches k8sClient's catch, which logs+ // "runtime reconciliation blocked" and returns BEFORE the managed Deployment+ // is created — so a fault in ONE binding kept a whole server's pod down,+ // healthy siblings included. #513 never asked for that: its acceptance is "no+ // permanent DNS failure line, no prune". D4 fail-static is the house rule for+ // a condition we cannot resolve this pass, and the transient branch already+ // obeys it. When there is a live policy to serve, serve it, report the fault,+ // and let the workload run on the last known-good egress.+ await seedAccumulatedPolicy({ lapsed: false })+ injectPostResolutionTypeError()++ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })++ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()++ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.status).toBe('False')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')+ })++ it('T8 still throws on a fault with nothing to serve (bootstrap)', async () => {+ // The other half of the same decision, pinned so it cannot be lost: with no+ // live policy there is no frozen set, so serving is not an option and the+ // reconcile must fail loud. Without this test, "stop throwing" could be+ // satisfied by never throwing at all.+ injectPostResolutionTypeError()++ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)+ })++ it('T9 reports the rejected binding alongside the fault instead of dropping it', async () => {+ // Review of #567, blocker 1. `failures` has exactly one sink — the message at+ // the end — and the fault branch throws before reaching it. A pass carrying+ // both loses the operator's own config error entirely: not in the status, not+ // in a log. Meanwhile cleanup has already deleted that binding's policy,+ // because a `failures.push(...); continue` never reaches+ // desiredPolicyNames.add. The operator is left with a de-authorized+ // destination, a status blaming an unrelated binding, and no trace of why.+ const twoBindings: McpServerCRD = {+ ...server,+ spec: {+ ...server.spec,+ egressBindings: [+ { dns: 'api.example.com', port: 443 },+ { dns: '*.wildcard.example.com', port: 443 },+ ],+ },+ }+ resolve4Mock.mockResolvedValue(rec('1.2.3.4'))+ await reconciler.reconcileExternalEgress(twoBindings, { isCurrent: () => true }).catch(() => {})+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()+ injectPostResolutionTypeError()++ await reconciler+ .reconcileExternalEgress(twoBindings, { isCurrent: () => true })+ .catch(() => undefined)++ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ const message = egress.at(-1)?.message ?? ''+ expect(message).toContain('api.example.com')+ expect(message).toMatch(/wildcard/)+ })+it('T5 keeps freezing the lapsed window on a real transient resolver code', async () => {// The gate has two exemptions and this pins the one the other tests do not// touch. Deleting `kind !== 'transient'` from the condition turns everydiff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex aa1825498..f5214db91 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -2067,7 +2067,7 @@ export class NetworkPolicyReconciler {// Issue #513: faults that are NOT a DNS condition. Kept separate from// `failures` because those are rejected INPUT (the operator's) while these are// OUR bug, and the two earn different reasons and different messages.- const reconcileFaults: Array<{ dns: string; err: unknown }> = []+ const reconcileFaults: Array<{ dns: string; err: unknown; blocking: boolean }> = []const resolvedAtMs = Date.now()const resolvedAt = new Date(resolvedAtMs).toISOString()@@ -2282,7 +2282,36 @@ export class NetworkPolicyReconciler {// the omission as "the operator removed it" and delete the very// policy this branch promises to leave untouched.desiredPolicyNames.add(name)- reconcileFaults.push({ dns: binding.dns ?? '(unnamed binding)', err })++ // D4 fail-static, applied to a fault exactly as the transient branch+ // applies it to a resolver outage: when there is a live policy, its+ // rules are what is enforced right now, so report them and let the+ // workload run on the last known-good egress. Read from the live+ // object, not recomputed — recomputing is what faulted.+ const enforced = (existingPolicy?.spec?.egress ?? [])+ .flatMap(rule => rule.to ?? [])+ .map(to => to.ipBlock?.cidr)+ .filter((cidr): cidr is string => typeof cidr === 'string')+ if (enforced.length > 0) {+ // Without this the binding vanishes from status.resolvedEgressIPs —+ // the audit record an operator reads to learn what is allowed —+ // while its /32s stay enforced. The status message would claim+ // "accumulated egress left untouched" and the record would disagree.+ resolvedEgressIPs.push({+ dns: binding.dns,+ ips: enforced.map(cidr => cidr.replace(/\/32$/, '')),+ resolvedAt,+ })+ }+ reconcileFaults.push({+ dns: binding.dns ?? '(unnamed binding)',+ err,+ // Blocking only when there is nothing to serve. A fault on a+ // binding that has never been written has no frozen set behind it,+ // so completing the pass would publish a server as reconciled while+ // one of its declared destinations was never enforced at all.+ blocking: enforced.length === 0,+ })continue}const accumulated = accumulateHostExactHostEgress({@@ -2379,20 +2408,42 @@ export class NetworkPolicyReconciler {// the spec is still de-authorized on a pass that ends in a fault.if (reconcileFaults.length > 0) {const first = reconcileFaults[0]- const message =+ const parts = [`External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +- `(not a resolver condition, accumulated egress left untouched): ${this.errorMessage(first.err)}`+ `(not a resolver condition, accumulated egress left untouched): ${this.errorMessage(first.err)}`,+ ]+ if (reconcileFaults.length > 1) {+ parts.push(`and ${reconcileFaults.length - 1} more binding(s) faulted`)+ }+ // `failures` has exactly one sink — the ExternalEgressRejected message+ // below — which this branch used to make unreachable. A pass carrying both+ // then lost the operator's own config error completely, while cleanup had+ // already deleted that binding's policy: a de-authorized destination, a+ // status naming an unrelated binding, and no trace of the real cause.+ if (failures.length > 0) parts.push(`rejected binding(s): ${failures.join('; ')}`)await this.writeExternalEgressStatus(server,- resolvedEgressIPs,+ this.sortResolvedEgressIPs(resolvedEgressIPs),'False','ExternalEgressReconcileFailed',- reconcileFaults.length > 1- ? `${message} (and ${reconcileFaults.length - 1} more binding(s) faulted)`- : message,+ parts.join('. '),isCurrent)- throw first.err+ // Fail-static, not fail-stop. The throw reaches k8sClient's catch, which+ // returns before the managed Deployment is created — so throwing for a+ // fault that HAS a frozen set to serve takes a whole server's pod down over+ // one binding, healthy siblings included, on a retry ladder that cannot fix+ // a deterministic fault. Throw only when nothing could be served.+ const blocking = reconcileFaults.find(fault => fault.blocking)+ if (blocking) throw blocking.err+ // Rejected input stays blocking as it always was, now that it has been said+ // out loud in the message above.+ if (failures.length > 0) {+ throw new Error(+ `External egress reconciliation failed for "${server.name}": ${failures.join('; ')}`+ )+ }+ return}if (failures.length > 0) {