Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): flag a retry when a NetworkPolicy replace conflicts twice

Alfredo Lopez05fd296c49bb
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index fceaf3e7b..c66f3301a 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.378",
+ "version": "0.1.379",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.378",
+ "version": "0.1.379",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 8e1497512..aec7ee57c 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.378",
+ "version": "0.1.379",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 2700a36a4..ac08baba2 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -212,7 +212,11 @@ const RUNTIME_HTTP_EGRESS_OWNED_ANNOTATIONS: ReadonlySet<string> = new Set([
type RunLaneNetworkPolicyApplyResult =
| { policy: string; action: 'created' | 'replaced' | 'unchanged' }
- | { policy: string; action: 'retry'; reason: 'terminating' | 'absent-after-write-conflict' }
+ | {
+ policy: string
+ action: 'retry'
+ reason: 'terminating' | 'absent-after-write-conflict' | 'replace-conflicted-twice'
+ }
| {
policy: string
action: 'conflict'
@@ -4824,7 +4828,16 @@ export class WorkflowReconciler {
return { policy: name, action: 'replaced' }
} catch (error: unknown) {
// A stale resourceVersion gets one re-read and re-decision.
- if (getErrorCode(error) !== 409 || attempt === 1) throw error
+ if (getErrorCode(error) !== 409) throw error
+ if (attempt === 1) {
+ // Contention, not a defect: another writer changed the policy again
+ // between the re-read and the retry. Leave it for a later pass.
+ this.log.warn(`NetworkPolicy "${name}" replace conflicted twice; retrying later`, {
+ namespace,
+ reason: 'replace-conflicted-twice',
+ })
+ return { policy: name, action: 'retry', reason: 'replace-conflicted-twice' }
+ }
}
existing = await this.readNetworkPolicyOrNull(name, namespace)
if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 7ae1e2473..0b651bf8e 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3255,6 +3255,68 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
).toHaveLength(0)
})
+
+ it('flags a retry, without failing, when a concurrent writer wins both replaces of a run-lane policy', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+ expect(first.workflowPhase).not.toBe('failed')
+ const contendedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
+ const contended = live.get(contendedKey)
+ expect(contended).toBeDefined()
+ contended!.spec = { ...contended!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
+ const liveResourceVersion = contended!.metadata!.resourceVersion
+
+ // Another writer updates the policy just before each of our replaces, so
+ // both carry a resourceVersion that is already stale.
+ const apiserverReplace = api.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ let concurrentWrites = 0
+ api.replaceNamespacedNetworkPolicy.mockImplementation(async arg => {
+ if (arg.name === 'test-wf-coord-to-wrc') {
+ concurrentWrites += 1
+ const current = live.get(contendedKey)!
+ live.set(contendedKey, {
+ ...current,
+ metadata: { ...current.metadata, resourceVersion: `concurrent-${concurrentWrites}` },
+ })
+ }
+ return apiserverReplace(arg)
+ })
+ api.readNamespacedNetworkPolicy.mockClear()
+ api.replaceNamespacedNetworkPolicy.mockClear()
+ logs.entries.length = 0
+
+ const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(
+ api.readNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.name === 'test-wf-coord-to-wrc'
+ )
+ ).toHaveLength(2)
+ expect(
+ api.replaceNamespacedNetworkPolicy.mock.calls
+ .filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
+ .map(([arg]) => arg.body.metadata?.resourceVersion)
+ ).toEqual([liveResourceVersion, 'concurrent-1'])
+ expect(second.phase).not.toBe('failed')
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(second.networkPolicyRetryPending).toBe(true)
+ expect(
+ logs.entries.some(
+ entry =>
+ entry.level === 'warn' &&
+ entry.reason === 'replace-conflicted-twice' &&
+ String(entry.msg).includes('test-wf-coord-to-wrc')
+ )
+ ).toBe(true)
+ } finally {
+ logs.restore()
+ }
+ })
})
it('resolves snippet MCP transport workloads into the mounted runner config', async () => {