Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): ignore egress state of run-lane policies the apply leaves unwritten
Alfredo Lopez3fedac127754
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex a3d6a34f7..f1e0f7720 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.390",+ "version": "0.1.391","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.390",+ "version": "0.1.391","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 0ba32fea0..634b901f2 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.390",+ "version": "0.1.391","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 971fd2a7d..52c051c4f 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -34,6 +34,7 @@ import {import {type NetworkPolicyConvergenceDecision,buildNetworkPolicyReplacement,+ classifyOwnerlessNetworkPolicyOwnership,decideNetworkPolicyConvergence,} from '../reconciler/networkPolicyConvergence'import {@@ -308,6 +309,12 @@ interface RuntimeHttpEgressPolicyState {annotations: Record<string, string>}+type RuntimeHttpEgressPolicyRead =+ | { kind: 'absent' }+ | { kind: 'foreign' }+ | { kind: 'terminating'; annotations: Record<string, string> }+ | { kind: 'live'; annotations: Record<string, string> }+type PublicHttpEgressClass = 'exact-host' | 'public-web'function delay(ms: number): Promise<void> {@@ -410,9 +417,10 @@ function latestDate(a: Date | undefined, b: Date | undefined): Date | undefined}/**- * The latest live resolved-at when every policy exists and already carries the- * computed CIDR state. Undefined when a policy is missing, the resolved set- * changed, or no live value parses, so the caller stamps the current time.+ * The latest live resolved-at when every policy the apply writes exists and+ * already carries the computed CIDR state. Undefined when a policy is missing,+ * the resolved set changed, or no live value parses, so the caller stamps the+ * current time.*/function unchangedRuntimeHttpEgressResolvedAt(liveAnnotationSets: Array<Record<string, string> | undefined>,@@ -3283,7 +3291,13 @@ export class WorkflowReconciler {// preserves the widest still-active overlap window before writing identical state back.// Taking the latest expiration avoids one policy shortening another policy's DNS rollover.for (const policyName of policyNames) {- const annotations = await this.readNetworkPolicyAnnotations(namespace, policyName)+ const live = await this.readRuntimeHttpEgressPolicy(namespace, policyName)+ // The apply leaves a foreign or terminating policy unwritten, so its+ // annotations stop following DNS. Merging them reopened the overlap for+ // its stale CIDR and re-stamped resolved-at on every pass; counting it as+ // missing would re-stamp too. It is left out of the state entirely.+ if (live.kind === 'foreign' || live.kind === 'terminating') continue+ const annotations = live.kind === 'live' ? live.annotations : undefinedliveAnnotationSets.push(annotations)if (!annotations) continue@@ -3371,8 +3385,12 @@ export class WorkflowReconciler {let prunedPrevious = falsefor (const policyName of policyNames) {- const annotations = await this.readNetworkPolicyAnnotations(namespace, policyName)- if (!annotations) continue+ const live = await this.readRuntimeHttpEgressPolicy(namespace, policyName)+ // A foreign policy's CIDRs are not this workflow's state. A terminating+ // one still counts: the prune only drops expired entries, and its apply+ // defers that policy and reports the pending retry.+ if (live.kind === 'absent' || live.kind === 'foreign') continue+ const annotations = live.annotationsfor (const cidr of parseTrustedRuntimeHttpEgressCidrsAnnotation(annotations[RUNTIME_HTTP_EGRESS_CURRENT_CIDRS_ANNOTATION]@@ -3456,17 +3474,24 @@ export class WorkflowReconciler {return this.runtimeHttpEgressOverlapMs * 2}- private async readNetworkPolicyAnnotations(+ /**+ * Read a run-lane policy and classify it the way applyNetworkPolicy will. The+ * run-lane policies carry no ownerReferences, so the ownership veto of+ * decideNetworkPolicyConvergence reduces to the ownerless classification.+ */+ private async readRuntimeHttpEgressPolicy(namespace: string,name: string- ): Promise<Record<string, string> | undefined> {- try {- const policy = await this.deps.networkingApi.readNamespacedNetworkPolicy({ namespace, name })- return policy.metadata?.annotations ?? {}- } catch (error: unknown) {- if (getErrorCode(error) === 404) return undefined- throw error- }+ ): Promise<RuntimeHttpEgressPolicyRead> {+ const policy = await this.readNetworkPolicyOrNull(name, namespace)+ if (!policy) return { kind: 'absent' }+ if (classifyOwnerlessNetworkPolicyOwnership(policy).kind !== 'owned') {+ return { kind: 'foreign' }+ }+ const annotations = policy.metadata?.annotations ?? {}+ return policy.metadata?.deletionTimestamp+ ? { kind: 'terminating', annotations }+ : { kind: 'live', annotations }}private annotateRuntimeHttpEgressPolicies(diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 39e6513cb..a639d5263 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -2380,6 +2380,117 @@ describe('WorkflowReconciler — reconcile loop', () => {}})+ it('keeps the CIDRs of a policy owned by another controller out of the DNS-failure prune', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>+ const livePolicies: Record<string, k8s.V1NetworkPolicy> = {+ // Ours: the rollover to .35 is done and its .34 overlap has expired.+ 'test-wf-coord-to-wrc': {+ metadata: {+ name: 'test-wf-coord-to-wrc',+ resourceVersion: 'rv-coord',+ annotations: {+ 'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',+ 'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',+ 'clerum.io/runtime-http-egress-previous-expires-at': '2000-01-01T00:00:00.000Z',+ },+ },+ },+ // Another controller's: frozen on the CIDR DNS no longer returns.+ 'test-wf-snippet-runner-egress': {+ metadata: {+ name: 'test-wf-snippet-runner-egress',+ resourceVersion: 'rv-runner',+ ownerReferences: [+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'another-controller',+ uid: 'foreign-owner-uid',+ controller: true,+ },+ ],+ annotations: {+ 'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',+ },+ },+ },+ }+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(+ async ({ name }: { name: string }) => {+ const found = livePolicies[name]+ if (!found) throw { code: 404 }+ return structuredClone(found)+ }+ )+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: networkingApi as never,+ resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,+ })+ )+ const spec = makeSpec({+ agent: undefined,+ // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.+ coordinatorImage: 'registry.example/custom-coordinator:1',+ runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },+ steps: [+ {+ id: 'snippet',+ run: {+ type: 'snippet',+ language: 'typescript',+ code: 'return await sdk.http.fetchJson("https://api.example.com/data")',+ capabilities: {+ http: { allowedHosts: ['api.example.com'] },+ },+ },+ },+ ],+ })++ await expect(+ reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )+ ).rejects.toThrow('ENOTFOUND')++ // Liveness: the prune read both policies and applied the runner, which+ // the ownership branch refused.+ const readNames = networkingApi.readNamespacedNetworkPolicy.mock.calls.map(+ ([arg]) => (arg as { name: string }).name+ )+ expect(readNames).toEqual(+ expect.arrayContaining(['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress'])+ )+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ entry.reason === 'owner-reference-mismatch' &&+ String(entry.msg).includes('test-wf-snippet-runner-egress')+ )+ ).toBe(true)++ const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)+ expect(replaced.map(arg => arg.name)).toEqual(['test-wf-coord-to-wrc'])+ expect(publicHttpEgressCidrs(replaced[0]!.body)).toEqual(['93.184.216.35/32'])+ expect(replaced[0]!.body.metadata?.annotations).toMatchObject({+ 'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',+ })+ expect(+ replaced[0]!.body.metadata?.annotations?.['clerum.io/runtime-http-egress-previous-cidrs']+ ).toBeUndefined()+ } finally {+ logs.restore()+ }+ })+it('creates runtime HTTP egress policies without previous overlap on first refresh', async () => {const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })@@ -3339,6 +3450,186 @@ describe('WorkflowReconciler — reconcile loop', () => {})}})++ describe('DNS rollover while the snippet runner policy is left unwritten', () => {+ const COORD = 'test-wf-coord-to-wrc'+ const RUNNER = 'test-wf-snippet-runner-egress'+ const OLD_CIDR = '93.184.216.34/32'+ const NEW_CIDR = '93.184.216.35/32'+ const CURRENT_CIDRS = 'clerum.io/runtime-http-egress-current-cidrs'+ const PREVIOUS_CIDRS = 'clerum.io/runtime-http-egress-previous-cidrs'+ const REFRESH_INTERVAL_MS = 30_000+ // 40 minutes of refreshes: eight times the 300 s overlap window.+ const REFRESHES_AFTER_ROLLOVER = 80++ /**+ * Create both policies on the old CIDR, freeze the runner policy, move+ * DNS to the new CIDR and refresh every 30 s. Returns the coordinator+ * replaces seen during the loop; the mocks and `logs` then hold only one+ * further refresh.+ */+ async function rolloverWithFrozenRunner(+ logs: ReturnType<typeof captureRunLaneNetworkPolicyLogs>,+ freeze: (runner: k8s.V1NetworkPolicy) => void+ ) {+ const apiserver = makeApiserverNetworkingApi()+ let resolved = [OLD_CIDR]+ const resolve = vi.fn(async () => resolved)+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: apiserver.api as never,+ resolveRuntimeHttpEgressCidrs: resolve,+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,+ })+ )+ // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.+ const spec: WorkflowRecipeSpec = {+ ...runtimeEgressSpec(),+ coordinatorImage: 'registry.example/custom-coordinator:1',+ }+ const refresh = () =>+ reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )+ const coordKey = apiserver.key('sandbox-recipes', COORD)+ const runnerKey = apiserver.key('sandbox-recipes', RUNNER)++ vi.useFakeTimers({ toFake: ['Date'] })+ let now = FIRST_PASS_AT.getTime()+ vi.setSystemTime(now)+ await refresh()+ for (const policyKey of [coordKey, runnerKey]) {+ expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(+ OLD_CIDR+ )+ }+ freeze(apiserver.live.get(runnerKey)!)++ resolved = [NEW_CIDR]+ apiserver.api.replaceNamespacedNetworkPolicy.mockClear()+ for (let pass = 0; pass < REFRESHES_AFTER_ROLLOVER; pass += 1) {+ now += REFRESH_INTERVAL_MS+ vi.setSystemTime(now)+ await refresh()+ }+ const coordReplacesDuringLoop = apiserver.api.replaceNamespacedNetworkPolicy.mock.calls+ .map(([arg]) => arg)+ .filter(arg => arg.name === COORD)++ apiserver.api.createNamespacedNetworkPolicy.mockClear()+ apiserver.api.readNamespacedNetworkPolicy.mockClear()+ apiserver.api.replaceNamespacedNetworkPolicy.mockClear()+ logs.entries.length = 0+ now += REFRESH_INTERVAL_MS+ vi.setSystemTime(now)+ const last = await refresh()+ return { ...apiserver, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last }+ }++ it('drops the old CIDR from the owned policy and stops replacing it when the sibling has a foreign owner', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =+ await rolloverWithFrozenRunner(logs, runner => {+ runner.metadata = {+ ...runner.metadata,+ ownerReferences: [+ {+ apiVersion: 'apps/v1',+ kind: 'Deployment',+ name: 'another-controller',+ uid: 'foreign-owner-uid',+ controller: true,+ },+ ],+ }+ })++ // Liveness: every pass resolved DNS; the last one read both policies+ // by name for the egress state and again for the apply, and reached+ // the ownership branch for the runner.+ expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)+ expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])+ expect(last).toEqual({+ conflicts: [{ policy: RUNNER, reason: 'owner-reference-mismatch' }],+ retryPending: false,+ })+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ entry.reason === 'owner-reference-mismatch' &&+ String(entry.msg).includes(RUNNER)+ )+ ).toBe(true)+ // The foreign policy keeps the old CIDR it was frozen with.+ expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)++ // One replace opens the overlap, one closes it once it expires.+ expect(coordReplacesDuringLoop).toHaveLength(2)+ const lastReplaced = coordReplacesDuringLoop.at(-1)!.body+ expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])+ expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()+ expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])++ expect(+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)+ ).toHaveLength(0)+ expect(+ logs.entries.filter(entry =>+ String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)+ )+ ).toHaveLength(1)+ } finally {+ logs.restore()+ }+ })++ it('drops the old CIDR from the owned policy and stops replacing it when the sibling is terminating', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =+ await rolloverWithFrozenRunner(logs, runner => {+ runner.metadata = {+ ...runner.metadata,+ deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),+ }+ })++ expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)+ expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])+ expect(last).toEqual({ conflicts: [], retryPending: true })+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ String(entry.msg).includes(`NetworkPolicy "${RUNNER}" is terminating`)+ )+ ).toBe(true)+ expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)++ expect(coordReplacesDuringLoop).toHaveLength(2)+ const lastReplaced = coordReplacesDuringLoop.at(-1)!.body+ expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])+ expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()+ expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])++ expect(+ api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)+ ).toHaveLength(0)+ expect(+ logs.entries.filter(entry =>+ String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)+ )+ ).toHaveLength(1)+ } finally {+ logs.restore()+ }+ })+ })})it('reports a foreign-owned run-lane policy as a condition and keeps the run alive', async () => {