Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(wrc): cover the run-lane NetworkPolicy apply paths the review found untested
Alfredo Lopezf8d78620707b
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex e6b088cff..79bb6a1c6 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.384",+ "version": "0.1.385","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.384",+ "version": "0.1.385","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex f5d5f6ea1..daaa3e2a9 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.384",+ "version": "0.1.385","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex a965df9fe..73d3accc6 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3148,9 +3148,7 @@ describe('WorkflowReconciler — reconcile loop', () => {vi.useRealTimers()})- async function refreshTwice(- resolveRuntimeHttpEgressCidrs: ReturnType<typeof vi.fn>- ): Promise<ReturnType<typeof makeApiserverNetworkingApi>> {+ async function refreshTwice(resolveRuntimeHttpEgressCidrs: ReturnType<typeof vi.fn>) {const apiserver = makeApiserverNetworkingApi()const reconciler = new WorkflowReconciler(makeDeps({@@ -3180,28 +3178,96 @@ describe('WorkflowReconciler — reconcile loop', () => {apiserver.api.readNamespacedNetworkPolicy.mockClear()apiserver.api.replaceNamespacedNetworkPolicy.mockClear()vi.setSystemTime(SECOND_PASS_AT)- await reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ const second = await reconciler.refreshRuntimeHttpEgressNetworkPolicies('sandbox-recipes','test-wf','uid-123',spec)- return apiserver+ return { ...apiserver, second }}it('keeps the live resolved-at and writes nothing when the resolved set is unchanged', async () => {- const resolve = vi.fn().mockResolvedValue(['93.184.216.34/32'])- const { api, live, key } = await refreshTwice(resolve)+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const resolve = vi.fn().mockResolvedValue(['93.184.216.34/32'])+ const { api, live, key, second } = await refreshTwice(resolve)++ expect(resolve).toHaveBeenCalledTimes(2)+ expect(readPolicyNames(api)).toEqual(expect.arrayContaining(RUNTIME_EGRESS_POLICIES))+ expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ for (const name of RUNTIME_EGRESS_POLICIES) {+ expect(+ live.get(key('sandbox-recipes', name))?.metadata?.annotations?.[RESOLVED_AT]+ ).toBe(FIRST_PASS_AT.toISOString())+ }+ // No write is also what a deferred retry produces. The Unchanged line,+ // which the first pass cannot emit because it creates the policy, and+ // the empty summary prove the second pass decided the policy converged.+ expect(second).toEqual({ conflicts: [], retryPending: false })+ for (const name of RUNTIME_EGRESS_POLICIES) {+ expect(+ logs.entries.filter(entry =>+ String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)+ )+ ).toHaveLength(1)+ }+ } finally {+ logs.restore()+ }+ })- expect(resolve).toHaveBeenCalledTimes(2)- expect(readPolicyNames(api)).toEqual(expect.arrayContaining(RUNTIME_EGRESS_POLICIES))- expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)- expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)- for (const name of RUNTIME_EGRESS_POLICIES) {- expect(live.get(key('sandbox-recipes', name))?.metadata?.annotations?.[RESOLVED_AT]).toBe(+ it('stamps the current time on a policy that is missing although its sibling is unchanged', async () => {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({+ networkingApi: apiserver.api as never,+ resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),+ config: { ...makeConfig(), enableSnippetRuntime: true } as never,+ })+ )+ // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.+ const spec: WorkflowRecipeSpec = {+ ...runtimeEgressSpec(),+ coordinatorImage: 'registry.example/custom-coordinator:1',+ }+ vi.useFakeTimers({ toFake: ['Date'] })+ vi.setSystemTime(FIRST_PASS_AT)+ await reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )+ const coordKey = apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')+ const runnerKey = apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress')+ for (const policyKey of [coordKey, runnerKey]) {+ expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(FIRST_PASS_AT.toISOString())}+ expect(apiserver.live.delete(coordKey)).toBe(true)++ apiserver.api.createNamespacedNetworkPolicy.mockClear()+ vi.setSystemTime(SECOND_PASS_AT)+ await reconciler.refreshRuntimeHttpEgressNetworkPolicies(+ 'sandbox-recipes',+ 'test-wf',+ 'uid-123',+ spec+ )++ // The unchanged sibling must not lend its old resolved-at to a policy+ // that did not exist: the recreated one carries the current time.+ const created = apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)+ expect(created.map(arg => arg.body.metadata?.name)).toEqual(['test-wf-coord-to-wrc'])+ expect(created[0]!.body.metadata?.annotations?.[RESOLVED_AT]).toBe(+ SECOND_PASS_AT.toISOString()+ )+ expect(apiserver.live.get(coordKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(+ SECOND_PASS_AT.toISOString()+ )})it('stamps the current time when the resolved set changes', async () => {@@ -3372,7 +3438,7 @@ describe('WorkflowReconciler — reconcile loop', () => {}})- describe('replace errors other than a conflict', () => {+ describe('replace failures after the read', () => {async function reconcileWithFailingReplace(failReplace: (arg: { name: string; namespace: string },@@ -3455,6 +3521,166 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(second.workflowPhase).toBe('failed')expect(second.message).toContain('HTTP-Code: 422')})++ it('flags a retry, without failing, when the policy is deleted after a replace conflict', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { second, reads, replaces } = await reconcileWithFailingReplace((arg, live) => {+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined+ // Another actor deletes the policy and our PUT loses the race, so+ // the re-read after the conflict finds nothing.+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)+ return Promise.reject({ code: 409 })+ })++ expect(reads).toHaveLength(2)+ expect(replaces).toHaveLength(1)+ expect(second.phase).not.toBe('failed')+ expect(second.workflowPhase).not.toBe('failed')+ expect(second.networkPolicyRetryPending).toBe(true)+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ String(entry.msg).includes('test-wf-coord-to-wrc') &&+ String(entry.msg).includes('vanished')+ )+ ).toBe(true)+ } finally {+ logs.restore()+ }+ })+ })++ describe('create conflicts, read errors and stale runtime state', () => {+ const CONTENDED = 'test-wf-coord-to-wrc'++ function callsFor<T extends { name?: string; body?: k8s.V1NetworkPolicy }>(mock: {+ mock: { calls: Array<[T]> }+ }): T[] {+ return mock.mock.calls+ .map(([arg]) => arg)+ .filter(arg => (arg.name ?? arg.body?.metadata?.name) === CONTENDED)+ }++ it('replaces the winner of a create race with the winner resourceVersion', async () => {+ const { api, live, key } = makeApiserverNetworkingApi()+ const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!+ let winnerResourceVersion: string | undefined+ api.createNamespacedNetworkPolicy.mockImplementation(async arg => {+ if (arg.body.metadata?.name === CONTENDED) {+ // Another writer creates a drifted copy between our 404 read and our POST.+ const winner = await apiserverCreate({+ namespace: arg.namespace,+ body: {+ ...arg.body,+ spec: { ...arg.body.spec!, podSelector: { matchLabels: { app: 'race-winner' } } },+ },+ })+ winnerResourceVersion = winner.metadata?.resourceVersion+ }+ return apiserverCreate(arg)+ })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ expect(result.workflowPhase).not.toBe('failed')+ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)+ const replaces = callsFor(api.replaceNamespacedNetworkPolicy)+ expect(replaces).toHaveLength(1)+ expect(winnerResourceVersion).toBeDefined()+ expect(replaces[0]!.body.metadata?.resourceVersion).toBe(winnerResourceVersion)+ expect(live.get(key('sandbox-recipes', CONTENDED))!.spec!.podSelector).not.toEqual({+ matchLabels: { app: 'race-winner' },+ })+ })++ it('flags a retry, without failing, when a create conflicts and the re-read finds nothing', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { api } = makeApiserverNetworkingApi()+ const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!+ api.createNamespacedNetworkPolicy.mockImplementation(async arg => {+ // The winner of the race is deleted again before our re-read.+ if (arg.body.metadata?.name === CONTENDED) throw { code: 409 }+ return apiserverCreate(arg)+ })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)+ expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)+ expect(result.phase).not.toBe('failed')+ expect(result.workflowPhase).not.toBe('failed')+ expect(result.networkPolicyRetryPending).toBe(true)+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ String(entry.msg).includes(CONTENDED) &&+ String(entry.msg).includes('vanished')+ )+ ).toBe(true)+ } finally {+ logs.restore()+ }+ })++ it('fails the run on a read error other than 404 and writes nothing for that policy', async () => {+ const { api } = makeApiserverNetworkingApi()+ const apiserverRead = api.readNamespacedNetworkPolicy.getMockImplementation()!+ api.readNamespacedNetworkPolicy.mockImplementation(async arg => {+ if (arg.name === CONTENDED) {+ throw Object.assign(new Error('HTTP-Code: 403 Message: forbidden'), { code: 403 })+ }+ return apiserverRead(arg)+ })+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ // A 403 is not absence: creating over it would hide a permission defect.+ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(0)+ expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)+ expect(result.workflowPhase).toBe('failed')+ expect(result.message).toContain('HTTP-Code: 403')+ })++ it('replaces a policy whose only drift is a stale runtime HTTP egress annotation', async () => {+ const { api, live, key } = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })+ const STALE = 'clerum.io/runtime-http-egress-previous-cidrs'++ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)+ expect(first.workflowPhase).not.toBe('failed')+ const stale = live.get(key('sandbox-recipes', CONTENDED))+ expect(stale).toBeDefined()+ expect(stale!.metadata?.annotations?.[STALE]).toBeUndefined()+ stale!.metadata = {+ ...stale!.metadata,+ annotations: { ...stale!.metadata?.annotations, [STALE]: '198.51.100.7/32' },+ }+ api.replaceNamespacedNetworkPolicy.mockClear()++ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ expect(second.workflowPhase).not.toBe('failed')+ const replaces = callsFor(api.replaceNamespacedNetworkPolicy)+ expect(replaces).toHaveLength(1)+ expect(replaces[0]!.body.metadata?.annotations?.[STALE]).toBeUndefined()+ expect(live.get(key('sandbox-recipes', CONTENDED))!.metadata?.annotations?.[STALE]).toBe(+ undefined+ )+ })})})