Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(egress): revoke a drifted policy the fault branch cannot rebuild
Alfredo Lopez0e71b66fd544
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex c48be2e26..367201d09 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.292",+ "version": "0.1.293","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.292",+ "version": "0.1.293","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex ef8938838..8b3b7266a 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.292",+ "version": "0.1.293","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.tsindex c2393ac66..10f12e0ac 100644--- a/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts+++ b/host-context-controller/src/networkPolicyReconciler.egressClassification.test.ts@@ -383,6 +383,69 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS conditexpect(reported).not.toContain('10.0.0.5')})+ it('T11 revokes the drifted policy instead of leaving the blocked range enforced', async () => {+ // Review of #567, R1-L1. T10 pins that a drifted policy is not SERVED and not+ // REPORTED. Neither of those stops it being ENFORCED: the fault branch adds+ // the binding to `desiredPolicyNames` before it knows whether the live object+ // is trustworthy, so `cleanupExternalEgress` reads it as still-desired and+ // keeps it, while the `continue` skips the rewrite. The private range then+ // stays in the dataplane for as long as the fault repeats — every 30s,+ // forever, for a deterministic fault.+ //+ // At the merge-base this same input self-healed: the fault was laundered into+ // the permanent branch, which rebuilt the policy from the annotation window+ // and wrote 1.2.3.4/32 back over the drift. So #567 made this input strictly+ // worse, and that is what this test exists to stop.+ //+ // The invariant both revisions share is that 10.0.0.5/32 does not survive the+ // pass. They reach it differently, and this test pins the mechanism THIS+ // branch can actually use: revocation. Rewriting — what the merge-base did —+ // is not available here, because rebuilding the policy means recomputing, and+ // recomputing is exactly what faulted. Revoking is fail-CLOSED: these are pure+ // allow policies (`policyTypes: ['Egress']` over an ipBlock list) under the+ // namespace L0 deny-all, so removing one closes that destination rather than+ // opening it, and none of them is a deny whose removal would open traffic.+ await seedAccumulatedPolicy({ lapsed: false })+ const items = (await mockApi.listNamespacedNetworkPolicy({ namespace: 'mcp-server' })) as {+ items: Array<{ metadata?: { name?: string }; spec?: k8s.V1NetworkPolicySpec }>+ }+ const drifted = JSON.parse(JSON.stringify(items)) as typeof items+ drifted.items[0].spec!.egress![0]!.to![0]!.ipBlock!.cidr = '10.0.0.5/32'+ const policyName = drifted.items[0].metadata!.name!+ mockApi.listNamespacedNetworkPolicy.mockResolvedValue(drifted)+ injectPostResolutionTypeError()++ // Liveness witness for the negative assertions below: the fault branch really+ // ran and really blocked. Without this, "the blocked range is not enforced"+ // would be satisfied just as well by a pass that never reached the branch.+ await expect(+ reconciler.reconcileExternalEgress(server, { isCurrent: () => true })+ ).rejects.toThrow(TypeError)++ // The revocation must happen BEFORE the throw. `cleanupExternalEgress` runs+ // after the binding loop and before the deferred fault is rethrown; a fix that+ // moved the throw earlier would restore the fail-loud-paid-with-fail-open+ // defect a previous round closed, and this assertion is what would catch it.+ expect(mockApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(+ expect.objectContaining({ name: policyName })+ )++ // Nothing rewrote it with the drift still in place, either.+ const rewritten = mockApi.replaceNamespacedNetworkPolicy.mock.calls+ .map(call => JSON.stringify((call[0] as { body?: unknown }).body))+ .join(' ')+ expect(rewritten).not.toContain('10.0.0.5')++ // And the operator is told what actually happened. The generic fault message+ // says "accumulated egress left untouched", which on this path would describe+ // the opposite of the pass — a status disagreeing with the dataplane is the+ // defect T7's sibling assertion was added to stop.+ const egress = statusConditions().filter(c => c.type === 'ExternalEgressReady')+ expect(egress.at(-1)?.reason).toBe('ExternalEgressReconcileFailed')+ expect(egress.at(-1)?.message).toContain('REVOKED')+ expect(egress.at(-1)?.message).not.toContain('left untouched')+ })+it('T8 still throws on a fault with nothing to serve (bootstrap)', async () => {// The other half of the same decision, pinned so it cannot be lost: with no// live policy there is no frozen set, so serving is not an option and the@@ -467,6 +530,20 @@ describe('#513: a non-DNS exception inside the resolve block is not a DNS condit// right — PERMANENT_DNS_CODES is shared with WRC, whose ui lane does reach// c-ares on the CRD pattern alone, and a validator is a weaker guarantee than// a gate that classifies correctly whatever arrives.+ //+ // NOT a RED-first test, and the PR body no longer claims it is (#567 review,+ // R1-M1): copy this file onto the merge-base and T4 passes, because there+ // `classifyDnsError` already defaults EBADNAME to 'permanent' and reaches the+ // same prune-and-reject outcome by the accident this PR exists to remove. No+ // assertion about EBADNAME can be red at the merge-base, since base and head+ // are observationally identical for this input; making it red would mean+ // changing what EBADNAME does, which is production code bent to a test.+ //+ // It is still live coverage, verified by mutation rather than asserted:+ // dropping `!isPermanentDnsCode(code)` from the gate kills it (along with+ // T3), and dropping 'EBADNAME' from PERMANENT_DNS_CODES kills T4 ALONE — no+ // other test in the HCC suite notices. That second mutation is what T4 is+ // here for.await seedAccumulatedPolicy({ lapsed: true })resolve4Mock.mockRejectedValueOnce(Object.assign(new Error('queryA EBADNAME api.example.com'), { code: 'EBADNAME' })diff --git a/host-context-controller/src/networkPolicyReconciler.ts b/host-context-controller/src/networkPolicyReconciler.tsindex 6d87ccadd..3225eaf61 100644--- a/host-context-controller/src/networkPolicyReconciler.ts+++ b/host-context-controller/src/networkPolicyReconciler.ts@@ -16,7 +16,12 @@import * as k8s from '@kubernetes/client-node'import * as dns from 'node:dns/promises'import { isIP } from 'node:net'-import { STATE_ANNOTATION, classifyDnsError, isPermanentDnsCode } from '@clerum/network-policy-core'+import {+ STATE_ANNOTATION,+ classifyDnsError,+ dnsCodeOf,+ isPermanentDnsCode,+} from '@clerum/network-policy-core'import { config } from './config'import {EXTERNAL_EGRESS_POLICY_TYPE,@@ -2067,7 +2072,12 @@ export class NetworkPolicyReconciler {// Issue #513: faults that are NOT a DNS condition. Kept separate from// `failures` because those are rejected INPUT (the operator's) while these are// OUR bug, and the two earn different reasons and different messages.- const reconcileFaults: Array<{ dns: string; err: unknown; blocking: boolean }> = []+ const reconcileFaults: Array<{+ dns: string+ err: unknown+ blocking: boolean+ revoked: boolean+ }> = []const resolvedAtMs = Date.now()const resolvedAt = new Date(resolvedAtMs).toISOString()@@ -2254,20 +2264,18 @@ export class NetworkPolicyReconciler {// operator's misconfiguration, and it must keep pruning and reporting// ExternalEgressRejected rather than blame the controller.//- // `code` is read off any object, so a rejection that is not an Error- // instance is still classified by its code rather than by how it was- // thrown. Not full parity with classifyDnsError, which also accepts a- // bare code string: a rejection whose value IS the string 'ENOTFOUND'- // classifies permanent there and yields no code here, so it lands in- // the fault branch. `node:dns` rejects with Error objects, so that- // shape does not occur in production; it is left explicit rather than- // handled, because inventing a string-rejection path would be code no- // caller reaches.+ // `dnsCodeOf` reads the code off any object, so a rejection that is not+ // an Error instance is still classified by its code rather than by how+ // it was thrown. It is deliberately not full parity with+ // classifyDnsError, which also accepts a bare code string: a rejection+ // whose value IS the string 'ENOTFOUND' classifies permanent there and+ // yields no code here, so it lands in the fault branch. `node:dns`+ // rejects with Error objects, so that shape does not occur in+ // production. Shared with WRC rather than unwrapped by hand in each+ // process — two hand-written extractions of the same field is the drift+ // surface network-policy-core exists to remove (#567 review, R1-M2).const kind = classifyDnsError(err)- const code =- typeof err === 'object' && err !== null- ? (err as NodeJS.ErrnoException).code- : undefined+ const code = dnsCodeOf(err)if (kind !== 'transient' && !isPermanentDnsCode(code)) {// Log before anything that can throw. writeExternalEgressStatus fails// on a stale uid/generation or a rejected patch, and the isCurrent@@ -2283,12 +2291,7 @@ export class NetworkPolicyReconciler {// desiredPolicyNames is only complete once the loop ends. Throwing// here would leave a de-authorized destination allowed for as long as// the fault persists — fail-loud paid for with fail-open.- // Still DESIRED: the binding is declared, we simply could not- // reconcile it this pass. Without this the cleanup below would read- // the omission as "the operator removed it" and delete the very- // policy this branch promises to leave untouched.- desiredPolicyNames.add(name)-+ //// D4 fail-static, applied to a fault exactly as the transient branch// applies it to a resolver outage: when there is a live policy, its// rules are what is enforced right now, so report them and let the@@ -2311,6 +2314,36 @@ export class NetworkPolicyReconciler {// known-good, so none of it is served and the fault blocks.const allowed = enforced.filter(cidr => isAllowedExternalEgressCidr(cidr))const trustworthy = allowed.length === enforced.length++ // Still DESIRED, but only while the live object is one we would keep.+ // The cleanup below deletes every external-egress policy NOT in this+ // set, so membership is the difference between "left untouched" and+ // "revoked", and the choice has to be made after the trustworthy+ // verdict, not before it (review of #567, R1-L1).+ //+ // Trustworthy — including the bootstrap case, where there is no live+ // policy at all and `enforced` is empty, so the filter is vacuously+ // total: the binding is declared, we simply could not reconcile it+ // this pass. Without the add, cleanup would read the omission as "the+ // operator removed it" and delete the very policy this branch+ // promises to leave untouched.+ //+ // Untrustworthy: the live policy carries a CIDR outside the allowed+ // ranges. Nothing else in the system repairs that — the safety lane+ // retains DNS policies modulo cidr and the orphan list is by label+ // without an ownership read — and this branch cannot rewrite it,+ // because rebuilding means recomputing and recomputing is what+ // faulted. Leaving it in the desired set is what kept a private range+ // enforced on every resync for as long as the fault lasted, which the+ // merge-base did not do: there the fault was laundered into the+ // permanent branch, which rewrote the policy from the annotation+ // window and dropped the drift. Omitting it hands the object to+ // cleanup, which runs BEFORE the deferred throw below, so the+ // revocation actually happens rather than being skipped by the+ // fail-loud exit. Revoking an allow policy under the namespace L0+ // deny-all closes that destination; it cannot open one.+ if (trustworthy) desiredPolicyNames.add(name)+if (trustworthy && enforced.length > 0) {// Without this the binding vanishes from status.resolvedEgressIPs —// the audit record an operator reads to learn what is allowed —@@ -2332,6 +2365,11 @@ export class NetworkPolicyReconciler {// so completing the pass would publish a server as reconciled while// one of its declared destinations was never enforced at all.blocking: !trustworthy || enforced.length === 0,+ // The message below otherwise tells the operator the accumulated+ // egress was left untouched. On an untrustworthy live policy it is+ // revoked instead, and a status that describes the opposite of what+ // the pass did is the same defect class this branch exists to close.+ revoked: !trustworthy,})continue}@@ -2436,7 +2474,11 @@ export class NetworkPolicyReconciler {const first = blocking ?? reconcileFaults[0]const parts = [`External egress reconciliation failed on binding "${first.dns}" with a non-DNS error ` +- `(not a resolver condition, accumulated egress left untouched): ${this.errorMessage(first.err)}`,+ `(not a resolver condition, ${+ first.revoked+ ? 'live policy REVOKED: it enforced an address outside the allowed ranges'+ : 'accumulated egress left untouched'+ }): ${this.errorMessage(first.err)}`,]if (reconcileFaults.length > 1) {parts.push(`and ${reconcileFaults.length - 1} more binding(s) faulted`)diff --git a/packages/network-policy-core/index.cjs b/packages/network-policy-core/index.cjsindex c1c2fcbcb..27286dcba 100644--- a/packages/network-policy-core/index.cjs+++ b/packages/network-policy-core/index.cjs@@ -104,10 +104,36 @@ const NO_RECORDS_DNS_CODES = Object.freeze(['ENODATA', 'ENOTFOUND'])*/const PERMANENT_DNS_CODES = Object.freeze([...NO_RECORDS_DNS_CODES, 'EBADNAME'])+/**+ * True only for a code in PERMANENT_DNS_CODES. Takes a CODE, not an Error —+ * `isPermanentDnsCode(err)` is always false, which would silently route every+ * permanent DNS answer into the caller's fault branch. Pair it with `dnsCodeOf`.+ */function isPermanentDnsCode(code) {return typeof code === 'string' && PERMANENT_DNS_CODES.includes(code)}+/**+ * The one extraction of `.code` off a rejection value (issue #567 review, R1-M2).+ * Both egress gates need "what code did this rejection carry", and a hand-written+ * unwrap in each process is the drift surface this package exists to remove;+ * `classifyDnsError` reads through here too, so there is a single answer.+ *+ * Deliberately narrower than `classifyDnsError`, which ALSO accepts a bare code+ * string: this returns undefined for a string input. A rejection whose value IS+ * the string 'ENOTFOUND' therefore yields no code and lands in the caller's fault+ * branch rather than being classified. `node:dns` rejects with Error objects, so+ * that shape does not occur; the asymmetry is kept because widening it would+ * invent a string-rejection path no caller reaches.+ *+ * Non-string `.code` values return undefined rather than propagating: the+ * declared type is `string | undefined` and every caller string-checks anyway.+ */+function dnsCodeOf(err) {+ if (typeof err !== 'object' || err === null) return undefined+ return typeof err.code === 'string' ? err.code : undefined+}+/** True for the two genuine no-records answers, false for every other code. */function isNoRecordsDnsCode(code) {return typeof code === 'string' && NO_RECORDS_DNS_CODES.includes(code)@@ -133,12 +159,7 @@ function isNoRecordsDnsCode(code) {* freezes the accumulated set.*/function classifyDnsError(errOrCode) {- const code =- typeof errOrCode === 'string'- ? errOrCode- : errOrCode && typeof errOrCode === 'object'- ? errOrCode.code- : undefined+ const code = typeof errOrCode === 'string' ? errOrCode : dnsCodeOf(errOrCode)return typeof code === 'string' && TRANSIENT_DNS_CODES.has(code) ? 'transient' : 'permanent'}@@ -479,4 +500,5 @@ module.exports = {PERMANENT_DNS_CODES,isPermanentDnsCode,isNoRecordsDnsCode,+ dnsCodeOf,}diff --git a/packages/network-policy-core/index.d.ts b/packages/network-policy-core/index.d.tsindex a4322484e..d8b2fc3be 100644--- a/packages/network-policy-core/index.d.ts+++ b/packages/network-policy-core/index.d.ts@@ -155,8 +155,25 @@ export declare function classifyDnsError(*/export declare const PERMANENT_DNS_CODES: readonly string[]+/**+ * True only for a code in PERMANENT_DNS_CODES. Takes a *code*, not an Error:+ * `isPermanentDnsCode(err)` is always false, which would silently route every+ * permanent DNS answer into the caller's fault branch. Pair it with `dnsCodeOf`.+ */export declare function isPermanentDnsCode(code: unknown): boolean+/**+ * Extract the DNS `.code` off a rejection value — the single implementation both+ * egress gates share instead of hand-writing the unwrap. Returns undefined for a+ * non-object, for a missing code, and for a non-string code.+ *+ * Narrower than `classifyDnsError` on purpose: a bare code STRING yields+ * undefined here, so a rejection whose value is the string 'ENOTFOUND' is not+ * classified as a DNS answer. `node:dns` rejects with Error objects, so that+ * shape does not occur in production.+ */+export declare function dnsCodeOf(err: unknown): string | undefined+/*** True for the two genuine no-records answers (ENODATA, ENOTFOUND) and false for* every other code, PERMANENT_DNS_CODES members included. Lets a caller word adiff --git a/packages/network-policy-core/index.test.cjs b/packages/network-policy-core/index.test.cjsindex 9b8f1a64b..14233ebcc 100644--- a/packages/network-policy-core/index.test.cjs+++ b/packages/network-policy-core/index.test.cjs@@ -368,6 +368,52 @@ test('#513: isPermanentDnsCode rejects what classifyDnsError merely defaults', (}})+// `isNoRecordsDnsCode` was reachable only through WRC before (#567 review nit).+// It splits PERMANENT_DNS_CODES in two so the operator gets the right words: a+// name with no records is a different event from one the resolver refused to+// query, and the two must not collapse back into "no A or AAAA records".+test('#567: isNoRecordsDnsCode is the empty-answer subset, EBADNAME excluded', () => {+ for (const code of ['ENODATA', 'ENOTFOUND']) {+ assert.equal(core.isNoRecordsDnsCode(code), true, `${code} is an empty answer`)+ assert.equal(core.isPermanentDnsCode(code), true, `${code} is also a permanent verdict`)+ }+ // The member that separates the two sets. If this ever returns true, WRC's+ // "the resolver refused to query it" branch goes unreachable and a malformed+ // name is reported as a name with no records.+ assert.equal(core.isNoRecordsDnsCode('EBADNAME'), false)+ assert.equal(core.isPermanentDnsCode('EBADNAME'), true)++ for (const code of [undefined, null, '', 'ESERVFAIL', 'EBADFAMILY', 123]) {+ assert.equal(core.isNoRecordsDnsCode(code), false, `${String(code)} is not an empty answer`)+ }+})++// ─── #567 R1-M2: one extraction of `.code`, shared by both egress gates ────────+// Each lane used to unwrap the rejection by hand — HCC with a typeof dance, WRC+// by reading `.code` straight off a value typed as an ErrnoException. Same field,+// two implementations, in two processes: the drift surface this package exists to+// remove. classifyDnsError reads through it too, so there is one answer.+test('#567: dnsCodeOf extracts the code off any object rejection', () => {+ assert.equal(core.dnsCodeOf(Object.assign(new Error('x'), { code: 'ENOTFOUND' })), 'ENOTFOUND')+ // Not an Error instance: classified by its code, not by how it was thrown.+ assert.equal(core.dnsCodeOf({ code: 'ESERVFAIL' }), 'ESERVFAIL')+})++test('#567: dnsCodeOf reports absence rather than inventing a code', () => {+ for (const value of [undefined, null, 42, new Error('no code'), {}, { code: 7 }, { code: null }]) {+ assert.equal(core.dnsCodeOf(value), undefined, `${String(value)} carries no string code`)+ }+})++test('#567: dnsCodeOf does NOT accept a bare code string, unlike classifyDnsError', () => {+ // The asymmetry is deliberate and load-bearing: a rejection whose VALUE is the+ // string 'ENOTFOUND' must not be treated as a DNS answer by the gates, so it+ // yields no code here and lands in the caller's fault branch. classifyDnsError+ // keeps its string form because callers pass it a code directly.+ assert.equal(core.dnsCodeOf('ENOTFOUND'), undefined)+ assert.equal(core.classifyDnsError('ENOTFOUND'), 'permanent')+ assert.equal(core.isPermanentDnsCode(core.dnsCodeOf('ENOTFOUND')), false)+})// ─── H-A: revocation/freeze keyed by (fqdn,port,protocol), not fqdn alone ──────// The state map is keyed by (fqdn,ip,port,protocol) but classification used to bediff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex fbee3b101..8549a3f01 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.340",+ "version": "0.1.341","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.340",+ "version": "0.1.341","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 7747bce78..0e09abecd 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.340",+ "version": "0.1.341","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/fqdnResolver.test.ts b/workflow-recipes/src/reconciler/fqdnResolver.test.tsindex 6ba0104bc..182ba8320 100644--- a/workflow-recipes/src/reconciler/fqdnResolver.test.ts+++ b/workflow-recipes/src/reconciler/fqdnResolver.test.ts@@ -245,6 +245,27 @@ describe('defaultFqdnLookup classification', () => {await expect(defaultFqdnLookup('h.example.com')).rejects.toMatchObject({cause: expect.objectContaining({ code }),})++ // #567 review, R1-M4. The message must NOT contain the substring+ // `egress resolution failed`: workflowRecipeReconciler keys on it to+ // exclude a message from the infra-retry reclassification, and this throw+ // is a raw fault that has not been classified by egressResolutionError()+ // at all, so it must stay eligible for the normal routing. The invariant+ // lived only in a comment above the throw until now.+ //+ // Captured rather than asserted with `.rejects.not.toThrow(...)`: that+ // form passes just as well when the promise never rejects, which would+ // mean this whole branch stopped throwing and the test still went green.+ // The positive assertions on the captured message are the liveness+ // witness that the throw really happened.+ const thrown = await defaultFqdnLookup('h.example.com').then(+ result => {+ throw new Error(`expected a throw, got ${JSON.stringify(result)}`)+ },+ (e: unknown) => e as Error+ )+ expect(thrown.message).toContain(code)+ expect(thrown.message).not.toContain('egress resolution failed')})diff --git a/workflow-recipes/src/reconciler/fqdnResolver.ts b/workflow-recipes/src/reconciler/fqdnResolver.tsindex 265ff332a..2a116bbc2 100644--- a/workflow-recipes/src/reconciler/fqdnResolver.ts+++ b/workflow-recipes/src/reconciler/fqdnResolver.ts@@ -2,6 +2,7 @@ import type { RecordWithTtl } from 'node:dns'import { resolve4, resolve6 } from 'node:dns/promises'import {classifyDnsError,+ dnsCodeOf,isNoRecordsDnsCode,isPermanentDnsCode,} from '@clerum/network-policy-core'@@ -189,11 +190,18 @@ export const defaultFqdnLookup: FqdnLookup = async host => {// message (walking `.cause` for socket codes), and this throw happens before// any policy write or delete, so the live NetworkPolicy is left untouched. The// message deliberately avoids the 'egress resolution failed' marker that catch- // matches on: a raw fault is not a DNS classification worth protecting.- const unclassified = rejections.find(e => !isPermanentDnsCode(e.code))+ // matches on: a raw fault is not a DNS classification worth protecting. That+ // absence is asserted in fqdnResolver.test.ts rather than left as prose.+ //+ // The code is read through the package's `dnsCodeOf` rather than off `.code`+ // directly, so both egress gates unwrap a rejection the same way (#567 review,+ // R1-M2). It also makes the read total: `rejections` is typed as+ // ErrnoException[] because the settle wrappers say so, not because anything+ // checked, so a non-object rejection would have thrown on the property access.+ const unclassified = rejections.find(e => !isPermanentDnsCode(dnsCodeOf(e)))if (unclassified) {throw new Error(- `DNS lookup for "${host}" failed with a non-DNS error (${unclassified.code ?? 'no code'}): ${describeRejection(unclassified)}`,+ `DNS lookup for "${host}" failed with a non-DNS error (${dnsCodeOf(unclassified) ?? 'no code'}): ${describeRejection(unclassified)}`,{ cause: unclassified })}@@ -203,11 +211,11 @@ export const defaultFqdnLookup: FqdnLookup = async host => {// code rather than testing for it: hardcoding 'EBADNAME' here would drift the// moment PERMANENT_DNS_CODES gains another member, and the message would fall// back to "no A or AAAA records" — the same lie this branch exists to remove.- const refused = rejections.find(e => !isNoRecordsDnsCode(e.code))+ const refused = rejections.find(e => !isNoRecordsDnsCode(dnsCodeOf(e)))if (refused) {return {kind: 'error',- error: `malformed hostname "${host}" (${refused.code}) — the resolver refused to query it`,+ error: `malformed hostname "${host}" (${dnsCodeOf(refused)}) — the resolver refused to query it`,}}@@ -238,12 +246,13 @@ export interface ResolveResult {* apply a stale or partial NetworkPolicy off the back of one. Each failure* carries a `retryable` flag: transient resolver failures (SERVFAIL/timeout)* are retryable, while a genuine no-records answer or a blocked-address- * rejection fails closed permanently. A lookup that cannot produce a DNS verdict- * at all throws instead of returning a failure, so the reconciler's existing- * error routing reports the real fault (issue #513).- * If any A record for a hostname resolves- * to a blocked range, the entire hostname fails closed and no public siblings- * are emitted.+ * rejection fails closed permanently. If any A record for a hostname resolves to+ * a blocked range, the entire hostname fails closed and no public siblings are+ * emitted.+ *+ * A lookup that cannot produce a DNS verdict at all throws instead of returning+ * a failure, so the reconciler's existing error routing reports the real fault+ * (issue #513).*/export async function resolveExternalEgress(externals: SandboxUiExternalEgress[],diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 5ac2d4777..b089a9871 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -9,6 +9,7 @@ import { shouldPatchRecipeStatus } from '../k8sClient'import { CronJobDef, WorkflowRecipeCRD, WorkflowRecipeGfsIntentSpec, WorkloadDef } from '../types'import { INHERITED_PARENT_RESOURCES_ANNOTATION } from '../workflow/childRecipeFactory'import { buildCoordinatorGfsNetworkPolicy } from '../workflow/networkPolicyFactory'+import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'@@ -107,6 +108,31 @@ const mockNetworkingApi = {const mockVerifyWorkflowRunProvenance = vi.fn().mockResolvedValue('verified')+// Lets the #513 tests below drive the REAL `defaultFqdnLookup` instead of+// fabricating the Error it produces (#567 review, R1-M3). The default+// implementation rejects loudly rather than returning something plausible: every+// other test in this file injects its own `fqdnLookup`, so a call arriving here+// means a test reached for real DNS by accident, and that should fail the test+// rather than resolve against the network or hang.+const { resolve4Mock, resolve6Mock } = vi.hoisted(() => ({+ resolve4Mock: vi.fn(),+ resolve6Mock: vi.fn(),+}))+vi.mock('node:dns/promises', () => ({+ resolve4: resolve4Mock,+ resolve6: resolve6Mock,+}))+// `vi.clearAllMocks()` in beforeEach clears CALLS, not implementations, so this+// default stands for the whole file; the #513 tests override it with+// `mockRejectedValueOnce` so the loud default is back on the next test.+const unexpectedDnsCall = (family: string) => async (host: string) => {+ throw new Error(+ `unexpected real DNS ${family} lookup for "${host}" — inject fqdnLookup in this test`+ )+}+resolve4Mock.mockImplementation(unexpectedDnsCall('A'))+resolve6Mock.mockImplementation(unexpectedDnsCall('AAAA'))+vi.mock('@kubernetes/client-node', () => ({KubeConfig: vi.fn().mockImplementation(() => ({makeApiClient: vi.fn().mockImplementation((ApiClass: unknown) => {@@ -2452,13 +2478,21 @@ describe('WorkflowRecipeReconciler', () => {],},})+ // The throw comes from the real producer, not from a fixture that imitates+ // it (#567 review, R1-M3). A hand-written Error pins only this test's own+ // guess at the message and `cause` shape, so a drift in `defaultFqdnLookup`+ // — different wording, a restructured cause, a change in describeRejection —+ // would leave this green while production routing broke. Mocking node:dns+ // one layer lower makes the producer emit the shape, so the two move+ // together or this test fails.+ resolve4Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ )+ resolve6Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })+ )const reconcilerWithLookup = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {- fqdnLookup: async () => {- throw new Error(- 'DNS lookup for "api.example.com" failed with a non-DNS error (EBADFAMILY): queryA EBADFAMILY',- { cause: Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' }) }- )- },+ fqdnLookup: host => defaultFqdnLookup(host),})const result = await reconcilerWithLookup.reconcile(recipe)@@ -2467,6 +2501,14 @@ describe('WorkflowRecipeReconciler', () => {expect(result.message).toContain('EBADFAMILY')// Not laundered into the operator-facing no-records wording.expect(result.message).not.toContain('no A or AAAA records')+ // And it arrived as a RAW FAULT, not as a classified egress-resolution+ // failure. This is the assertion that distinguishes the two routes: a+ // `kind: 'error'` result travels through egressResolutionError(), which+ // stamps `egress resolution failed` into the message, while a throw bypasses+ // that classification entirely. Without this, restoring the #513 defect —+ // returning an error instead of throwing — leaves the test green, because+ // both routes end in `failed` with the code somewhere in the message.+ expect(result.message).not.toContain('egress resolution failed')const egressPolicy = expect.objectContaining({body: expect.objectContaining({@@ -3985,14 +4027,18 @@ describe('WorkflowRecipeReconciler', () => {// records", and that only helps if BOTH call sites land somewhere honest:// the terminal `failed` phase carrying the real code, reached before any// write or delete, so a live `ui-egress-*` policy is left as it was.+ // Driven through the real `defaultFqdnLookup` for the same reason as the+ // workload-lane test above (#567 review, R1-M3): the throw must be the one+ // production emits, not a copy of it maintained by hand in the test.+ resolve4Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' })+ )+ resolve6Mock.mockRejectedValueOnce(+ Object.assign(new Error('queryAaaa EBADFAMILY'), { code: 'EBADFAMILY' })+ )const kc = new k8s.KubeConfig()const rec = new WorkflowRecipeReconciler(kc, undefined, {- fqdnLookup: async () => {- throw new Error(- 'DNS lookup for "api.stripe.com" failed with a non-DNS error (EBADFAMILY): queryA EBADFAMILY',- { cause: Object.assign(new Error('queryA EBADFAMILY'), { code: 'EBADFAMILY' }) }- )- },+ fqdnLookup: host => defaultFqdnLookup(host),})const result = await rec.reconcile(@@ -4003,6 +4049,14 @@ describe('WorkflowRecipeReconciler', () => {expect(result.message).toContain('EBADFAMILY')// Not laundered into the operator-facing no-records wording.expect(result.message).not.toContain('no A or AAAA records')+ // And it arrived as a RAW FAULT, not as a classified egress-resolution+ // failure. This is the assertion that distinguishes the two routes: a+ // `kind: 'error'` result travels through egressResolutionError(), which+ // stamps `egress resolution failed` into the message, while a throw bypasses+ // that classification entirely. Without this, restoring the #513 defect —+ // returning an error instead of throwing — leaves the test green, because+ // both routes end in `failed` with the code somewhere in the message.+ expect(result.message).not.toContain('egress resolution failed')expect(createdPolicy('ui-egress-test-recipe')).toBeUndefined()expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(