Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): flag a retry when a NetworkPolicy replace conflicts twice
Alfredo Lopez05fd296c49bb
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex fceaf3e7b..c66f3301a 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.378",+ "version": "0.1.379","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.378",+ "version": "0.1.379","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 8e1497512..aec7ee57c 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.378",+ "version": "0.1.379","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 2700a36a4..ac08baba2 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -212,7 +212,11 @@ const RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS: ReadonlySet<string> = new Set([type RunLaneNetworkPolicyApplyResult =| { policy: string; action: 'created' | 'replaced' | 'unchanged' }- | { policy: string; action: 'retry'; reason: 'terminating' | 'absent-after-write-conflict' }+ | {+ policy: string+ action: 'retry'+ reason: 'terminating' | 'absent-after-write-conflict' | 'replace-conflicted-twice'+ }| {policy: stringaction: 'conflict'@@ -4824,7 +4828,16 @@ export class WorkflowReconciler {return { policy: name, action: 'replaced' }} catch (error: unknown) {// A stale resourceVersion gets one re-read and re-decision.- if (getErrorCode(error) !== 409 || attempt === 1) throw error+ if (getErrorCode(error) !== 409) throw error+ if (attempt === 1) {+ // Contention, not a defect: another writer changed the policy again+ // between the re-read and the retry. Leave it for a later pass.+ this.log.warn(`NetworkPolicy "${name}" replace conflicted twice; retrying later`, {+ namespace,+ reason: 'replace-conflicted-twice',+ })+ return { policy: name, action: 'retry', reason: 'replace-conflicted-twice' }+ }}existing = await this.readNetworkPolicyOrNull(name, namespace)if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 7ae1e2473..0b651bf8e 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3255,6 +3255,68 @@ describe('WorkflowReconciler — reconcile loop', () => {)).toHaveLength(0)})++ it('flags a retry, without failing, when a concurrent writer wins both replaces of a run-lane policy', async () => {+ const logs = captureRunLaneNetworkPolicyLogs()+ try {+ const { api, live, key } = makeApiserverNetworkingApi()+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })++ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)+ expect(first.workflowPhase).not.toBe('failed')+ const contendedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')+ const contended = live.get(contendedKey)+ expect(contended).toBeDefined()+ contended!.spec = { ...contended!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }+ const liveResourceVersion = contended!.metadata!.resourceVersion++ // Another writer updates the policy just before each of our replaces, so+ // both carry a resourceVersion that is already stale.+ const apiserverReplace = api.replaceNamespacedNetworkPolicy.getMockImplementation()!+ let concurrentWrites = 0+ api.replaceNamespacedNetworkPolicy.mockImplementation(async arg => {+ if (arg.name === 'test-wf-coord-to-wrc') {+ concurrentWrites += 1+ const current = live.get(contendedKey)!+ live.set(contendedKey, {+ ...current,+ metadata: { ...current.metadata, resourceVersion: `concurrent-${concurrentWrites}` },+ })+ }+ return apiserverReplace(arg)+ })+ api.readNamespacedNetworkPolicy.mockClear()+ api.replaceNamespacedNetworkPolicy.mockClear()+ logs.entries.length = 0++ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)++ expect(+ api.readNamespacedNetworkPolicy.mock.calls.filter(+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'+ )+ ).toHaveLength(2)+ expect(+ api.replaceNamespacedNetworkPolicy.mock.calls+ .filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')+ .map(([arg]) => arg.body.metadata?.resourceVersion)+ ).toEqual([liveResourceVersion, 'concurrent-1'])+ expect(second.phase).not.toBe('failed')+ expect(second.workflowPhase).not.toBe('failed')+ expect(second.networkPolicyRetryPending).toBe(true)+ expect(+ logs.entries.some(+ entry =>+ entry.level === 'warn' &&+ entry.reason === 'replace-conflicted-twice' &&+ String(entry.msg).includes('test-wf-coord-to-wrc')+ )+ ).toBe(true)+ } finally {+ logs.restore()+ }+ })})it('resolves snippet MCP transport workloads into the mounted runner config', async () => {