Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): flag a retry when a run-lane NetworkPolicy is deleted before the replace
Alfredo Lopez3684cc08a199
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex c66f3301a..fcc15d836 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.379",+ "version": "0.1.380","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.379",+ "version": "0.1.380","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex aec7ee57c..912104a91 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.379",+ "version": "0.1.380","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex ac08baba2..181592bd7 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -4767,9 +4767,11 @@ export class WorkflowReconciler {}/**- * Read first, then write only what the live object lacks. Ownership conflicts- * and terminating objects are returned, not thrown: the workflow reconcile- * catch turns every non-transport error into a terminal `failed`.+ * Read first, then write only what the live object lacks. Ownership conflicts,+ * terminating objects, a policy deleted before the PUT and a replace that+ * conflicts twice are returned, not thrown: the workflow reconcile catch turns+ * every non-transport error into a terminal `failed`. Any other API error+ * still propagates.*/private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy@@ -4827,8 +4829,11 @@ export class WorkflowReconciler {this.log.info(`Updated NetworkPolicy "${name}"`, { namespace, reason: decision.reason })return { policy: name, action: 'replaced' }} catch (error: unknown) {+ const code = getErrorCode(error)+ // Deleted between the read and the PUT: the next pass recreates it.+ if (code === 404) return this.networkPolicyAbsentAfterConflict(name, namespace)// A stale resourceVersion gets one re-read and re-decision.- if (getErrorCode(error) !== 409) throw error+ if (code !== 409) throw errorif (attempt === 1) {// Contention, not a defect: another writer changed the policy again// between the re-read and the retry. Leave it for a later pass.diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 0b651bf8e..732dd443f 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3317,6 +3317,91 @@ describe('WorkflowReconciler — reconcile loop', () => {logs.restore()}})++ describe('replace errors other than a conflict', () => {+ async function reconcileWithFailingReplace(+ failReplace: (+ arg: { name: string; namespace: string },+ live: Map<string, k8s.V1NetworkPolicy>+ ) => Promise<never> | undefined+ ) {+ const apiserver = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(+ makeDeps({ networkingApi: apiserver.api as never })+ )+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)+ expect(first.workflowPhase).not.toBe('failed')+ const drifted = apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))+ expect(drifted).toBeDefined()+ drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }++ const apiserverReplace =+ apiserver.api.replaceNamespacedNetworkPolicy.getMockImplementation()!+ apiserver.api.replaceNamespacedNetworkPolicy.mockImplementation(+ async arg => failReplace(arg, apiserver.live) ?? apiserverReplace(arg)+ )+ apiserver.api.readNamespacedNetworkPolicy.mockClear()+ apiserver.api.replaceNamespacedNetworkPolicy.mockClear()++ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)+ const contendedCalls = (mock: typeof apiserver.api.readNamespacedNetworkPolicy) =>+ mock.mock.calls.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')+ return {+ second,+ reads: contendedCalls(apiserver.api.readNamespacedNetworkPolicy),+ replaces: apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'+ ),+ }+ }++ it('flags a retry, without failing, when the policy is deleted between the read and the replace', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { second, reads, replaces } = await reconcileWithFailingReplace((arg, live) => {+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined+ // Another actor deletes the policy after our read, so the+ // apiserver double answers the PUT with 404.+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)+ return undefined+ })++ expect(reads).toHaveLength(1)+ expect(replaces).toHaveLength(1)+ expect(second.phase).not.toBe('failed')+ expect(second.workflowPhase).not.toBe('failed')+ expect(second.networkPolicyRetryPending).toBe(true)+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ String(entry.msg).includes('test-wf-coord-to-wrc') &&+ String(entry.msg).includes('vanished')+ )+ ).toBe(true)+ } finally {+ logs.restore()+ }+ })++ it('propagates a replace error that is neither a conflict nor a deletion', async () => {+ const { second, reads, replaces } = await reconcileWithFailingReplace(arg => {+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined+ return Promise.reject(+ Object.assign(new Error('HTTP-Code: 422 Message: NetworkPolicy is invalid'), {+ code: 422,+ })+ )+ })++ expect(reads).toHaveLength(1)+ expect(replaces).toHaveLength(1)+ expect(second.workflowPhase).toBe('failed')+ expect(second.message).toContain('HTTP-Code: 422')+ })+ })})it('resolves snippet MCP transport workloads into the mounted runner config', async () => {