Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): flag a retry when a run-lane NetworkPolicy is deleted before the replace

Alfredo Lopez3684cc08a199
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index c66f3301a..fcc15d836 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.379",
+ "version": "0.1.380",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.379",
+ "version": "0.1.380",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index aec7ee57c..912104a91 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.379",
+ "version": "0.1.380",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index ac08baba2..181592bd7 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -4767,9 +4767,11 @@ export class WorkflowReconciler {
}
/**
- * Read first, then write only what the live object lacks. Ownership conflicts
- * and terminating objects are returned, not thrown: the workflow reconcile
- * catch turns every non-transport error into a terminal `failed`.
+ * Read first, then write only what the live object lacks. Ownership conflicts,
+ * terminating objects, a policy deleted before the PUT and a replace that
+ * conflicts twice are returned, not thrown: the workflow reconcile catch turns
+ * every non-transport error into a terminal `failed`. Any other API error
+ * still propagates.
*/
private async applyNetworkPolicy(
policy: k8s.V1NetworkPolicy
@@ -4827,8 +4829,11 @@ export class WorkflowReconciler {
this.log.info(`Updated NetworkPolicy "${name}"`, { namespace, reason: decision.reason })
return { policy: name, action: 'replaced' }
} catch (error: unknown) {
+ const code = getErrorCode(error)
+ // Deleted between the read and the PUT: the next pass recreates it.
+ if (code === 404) return this.networkPolicyAbsentAfterConflict(name, namespace)
// A stale resourceVersion gets one re-read and re-decision.
- if (getErrorCode(error) !== 409) throw error
+ if (code !== 409) throw error
if (attempt === 1) {
// Contention, not a defect: another writer changed the policy again
// between the re-read and the retry. Leave it for a later pass.
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 0b651bf8e..732dd443f 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3317,6 +3317,91 @@ describe('WorkflowReconciler — reconcile loop', () => {
logs.restore()
}
})
+
+ describe('replace errors other than a conflict', () => {
+ async function reconcileWithFailingReplace(
+ failReplace: (
+ arg: { name: string; namespace: string },
+ live: Map<string, k8s.V1NetworkPolicy>
+ ) => Promise<never> | undefined
+ ) {
+ const apiserver = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(
+ makeDeps({ networkingApi: apiserver.api as never })
+ )
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ expect(first.workflowPhase).not.toBe('failed')
+ const drifted = apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
+ expect(drifted).toBeDefined()
+ drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
+
+ const apiserverReplace =
+ apiserver.api.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ apiserver.api.replaceNamespacedNetworkPolicy.mockImplementation(
+ async arg => failReplace(arg, apiserver.live) ?? apiserverReplace(arg)
+ )
+ apiserver.api.readNamespacedNetworkPolicy.mockClear()
+ apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
+
+ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ const contendedCalls = (mock: typeof apiserver.api.readNamespacedNetworkPolicy) =>
+ mock.mock.calls.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
+ return {
+ second,
+ reads: contendedCalls(apiserver.api.readNamespacedNetworkPolicy),
+ replaces: apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'
+ ),
+ }
+ }
+
+ it('flags a retry, without failing, when the policy is deleted between the read and the replace', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ const { second, reads, replaces } = await reconcileWithFailingReplace((arg, live) => {
+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined
+ // Another actor deletes the policy after our read, so the
+ // apiserver double answers the PUT with 404.
+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
+ return undefined
+ })
+
+ expect(reads).toHaveLength(1)
+ expect(replaces).toHaveLength(1)
+ expect(second.phase).not.toBe('failed')
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(second.networkPolicyRetryPending).toBe(true)
+ expect(
+ logs.entries.some(
+ entry =>
+ entry.level === 'warn' &&
+ String(entry.msg).includes('test-wf-coord-to-wrc') &&
+ String(entry.msg).includes('vanished')
+ )
+ ).toBe(true)
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('propagates a replace error that is neither a conflict nor a deletion', async () => {
+ const { second, reads, replaces } = await reconcileWithFailingReplace(arg => {
+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined
+ return Promise.reject(
+ Object.assign(new Error('HTTP-Code: 422 Message: NetworkPolicy is invalid'), {
+ code: 422,
+ })
+ )
+ })
+
+ expect(reads).toHaveLength(1)
+ expect(replaces).toHaveLength(1)
+ expect(second.workflowPhase).toBe('failed')
+ expect(second.message).toContain('HTTP-Code: 422')
+ })
+ })
})
it('resolves snippet MCP transport workloads into the mounted runner config', async () => {