Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(egress): fail-static on a fault with a live policy, and stop dropping rejections

Alfredo Lopez043593ede1c3
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 6704209ff..3929e0371 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index ac6bbd4bb..2b103109a 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.288",
+ "version": "0.1.289",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
index 5e40184da..f182dfb3e 100644
--- a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
+++ b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts
@@ -223,9 +223,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
injectPostResolutionTypeError()
- await expect(
- reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- ).rejects.toThrow(TypeError)
+ // No throw: there is a live policy, so the fault is served fail-static (T7).
+ // Everything else this test pins is unchanged.
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
// The operator must not be pointed at DNS for a fault in the reconciler.
expect(warn.mock.calls.flat().join('\n')).not.toContain('permanent DNS failure')
@@ -247,9 +247,9 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
await seedAccumulatedPolicy({ lapsed: true })
injectPostResolutionTypeError()
- await expect(
- reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
- ).rejects.toThrow(TypeError)
+ // No throw: the policy is live (its window lapsed, the object did not), so
+ // the fault is served fail-static (T7). The prune assertion is the point.
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
// Pre-fix this prunes the lapsed set to zero, hits the `continue`, and the
// policy is swept as undesired. D4 fail-static must hold for a fault that is
@@ -310,9 +310,11 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
// The operator removes a.example.com; b.example.com faults after resolution.
injectPostResolutionTypeError()
- await expect(
- reconciler.reconcileExternalEgress(bindings(['b.example.com']), { isCurrent: () => true })
- ).rejects.toThrow(TypeError)
+ // No throw: b.example.com has a live policy, so the fault is served
+ // fail-static (T7). What this test pins is that the cleanup still ran.
+ await reconciler.reconcileExternalEgress(bindings(['b.example.com']), {
+ isCurrent: () => true,
+ })
// Exactly one delete, and it is A's. `toHaveBeenCalledWith` alone is a
// membership check, not an exclusivity one: it stays green when the cleanup
@@ -328,6 +330,73 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit
)
})
+ it('T7 serves the frozen set and does NOT block the workload when a fault has a live policy', async () => {
+ // Review of #567, major 3. The throw reaches k8sClient's catch, which logs
+ // "runtime reconciliation blocked" and returns BEFORE the managed Deployment
+ // is created — so a fault in ONE binding kept a whole server's pod down,
+ // healthy siblings included. #513 never asked for that: its acceptance is "no
+ // permanent DNS failure line, no prune". D4 fail-static is the house rule for
+ // a condition we cannot resolve this pass, and the transient branch already
+ // obeys it. When there is a live policy to serve, serve it, report the fault,
+ // and let the workload run on the last known-good egress.
+ await seedAccumulatedPolicy({ lapsed: false })
+ injectPostResolutionTypeError()
+
+ await reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+
+ expect(mockApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ expect(egress.at(-1)?.status).toBe('False')
+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')
+ })
+
+ it('T8 still throws on a fault with nothing to serve (bootstrap)', async () => {
+ // The other half of the same decision, pinned so it cannot be lost: with no
+ // live policy there is no frozen set, so serving is not an option and the
+ // reconcile must fail loud. Without this test, "stop throwing" could be
+ // satisfied by never throwing at all.
+ injectPostResolutionTypeError()
+
+ await expect(
+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })
+ ).rejects.toThrow(TypeError)
+ })
+
+ it('T9 reports the rejected binding alongside the fault instead of dropping it', async () => {
+ // Review of #567, blocker 1. `failures` has exactly one sink — the message at
+ // the end — and the fault branch throws before reaching it. A pass carrying
+ // both loses the operator's own config error entirely: not in the status, not
+ // in a log. Meanwhile cleanup has already deleted that binding's policy,
+ // because a `failures.push(...); continue` never reaches
+ // desiredPolicyNames.add. The operator is left with a de-authorized
+ // destination, a status blaming an unrelated binding, and no trace of why.
+ const twoBindings: McpServerCRD = {
+ ...server,
+ spec: {
+ ...server.spec,
+ egressBindings: [
+ { dns: 'api.example.com', port: 443 },
+ { dns: '*.wildcard.example.com', port: 443 },
+ ],
+ },
+ }
+ resolve4Mock.mockResolvedValue(rec('1.2.3.4'))
+ await reconciler.reconcileExternalEgress(twoBindings, { isCurrent: () => true }).catch(() => {})
+ mockCustomApi.patchNamespacedCustomObjectStatus.mockClear()
+ injectPostResolutionTypeError()
+
+ await reconciler
+ .reconcileExternalEgress(twoBindings, { isCurrent: () => true })
+ .catch(() => undefined)
+
+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')
+ const message = egress.at(-1)?.message ?? ''
+ expect(message).toContain('api.example.com')
+ expect(message).toMatch(/wildcard/)
+ })
+
it('T5 keeps freezing the lapsed window on a real transient resolver code', async () => {
// The gate has two exemptions and this pins the one the other tests do not
// touch. Deleting `kind !== 'transient'` from the condition turns every
diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.ts
index aa1825498..f5214db91 100644
--- a/host-context-controller/src/networkPolicyReconciler.ts
+++ b/host-context-controller/src/networkPolicyReconciler.ts
@@ -2067,7 +2067,7 @@ export class NetworkPolicyReconciler {
// Issue #513: faults that are NOT a DNS condition. Kept separate from
// `failures` because those are rejected INPUT (the operator's) while these are
// OUR bug, and the two earn different reasons and different messages.
- const reconcileFaults: Array<{ dns: string; err: unknown }> = []
+ const reconcileFaults: Array<{ dns: string; err: unknown; blocking: boolean }> = []
const resolvedAtMs = Date.now()
const resolvedAt = new Date(resolvedAtMs).toISOString()
@@ -2282,7 +2282,36 @@ export class NetworkPolicyReconciler {
// the omission as "the operator removed it" and delete the very
// policy this branch promises to leave untouched.
desiredPolicyNames.add(name)
- reconcileFaults.push({ dns: binding.dns ?? '(unnamed binding)', err })
+
+ // D4 fail-static, applied to a fault exactly as the transient branch
+ // applies it to a resolver outage: when there is a live policy, its
+ // rules are what is enforced right now, so report them and let the
+ // workload run on the last known-good egress. Read from the live
+ // object, not recomputed — recomputing is what faulted.
+ const enforced = (existingPolicy?.spec?.egress ?? [])
+ .flatMap(rule => rule.to ?? [])
+ .map(to => to.ipBlock?.cidr)
+ .filter((cidr): cidr is string => typeof cidr === 'string')
+ if (enforced.length > 0) {
+ // Without this the binding vanishes from status.resolvedEgressIPs —
+ // the audit record an operator reads to learn what is allowed —
+ // while its /32s stay enforced. The status message would claim
+ // "accumulated egress left untouched" and the record would disagree.
+ resolvedEgressIPs.push({
+ dns: binding.dns,
+ ips: enforced.map(cidr => cidr.replace(/\/32$/, '')),
+ resolvedAt,
+ })
+ }
+ reconcileFaults.push({
+ dns: binding.dns ?? '(unnamed binding)',
+ err,
+ // Blocking only when there is nothing to serve. A fault on a
+ // binding that has never been written has no frozen set behind it,
+ // so completing the pass would publish a server as reconciled while
+ // one of its declared destinations was never enforced at all.
+ blocking: enforced.length === 0,
+ })
continue
}
const accumulated = accumulateHostExactHostEgress({
@@ -2379,20 +2408,42 @@ export class NetworkPolicyReconciler {
// the spec is still de-authorized on a pass that ends in a fault.
if (reconcileFaults.length > 0) {
const first = reconcileFaults[0]
- const message =
+ const parts = [
`External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +
- `(not a resolver condition, accumulated egress left untouched): ${this.errorMessage(first.err)}`
+ `(not a resolver condition, accumulated egress left untouched): ${this.errorMessage(first.err)}`,
+ ]
+ if (reconcileFaults.length > 1) {
+ parts.push(`and ${reconcileFaults.length - 1} more binding(s) faulted`)
+ }
+ // `failures` has exactly one sink — the ExternalEgressRejected message
+ // below — which this branch used to make unreachable. A pass carrying both
+ // then lost the operator's own config error completely, while cleanup had
+ // already deleted that binding's policy: a de-authorized destination, a
+ // status naming an unrelated binding, and no trace of the real cause.
+ if (failures.length > 0) parts.push(`rejected binding(s): ${failures.join('; ')}`)
await this.writeExternalEgressStatus(
server,
- resolvedEgressIPs,
+ this.sortResolvedEgressIPs(resolvedEgressIPs),
'False',
'ExternalEgressReconcileFailed',
- reconcileFaults.length > 1
- ? `${message} (and ${reconcileFaults.length - 1} more binding(s) faulted)`
- : message,
+ parts.join('. '),
isCurrent
)
- throw first.err
+ // Fail-static, not fail-stop. The throw reaches k8sClient's catch, which
+ // returns before the managed Deployment is created — so throwing for a
+ // fault that HAS a frozen set to serve takes a whole server's pod down over
+ // one binding, healthy siblings included, on a retry ladder that cannot fix
+ // a deterministic fault. Throw only when nothing could be served.
+ const blocking = reconcileFaults.find(fault => fault.blocking)
+ if (blocking) throw blocking.err
+ // Rejected input stays blocking as it always was, now that it has been said
+ // out loud in the message above.
+ if (failures.length > 0) {
+ throw new Error(
+ `External egress reconciliation failed for "${server.name}": ${failures.join('; ')}`
+ )
+ }
+ return
}
if (failures.length > 0) {