Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): recreate an absent run-lane NetworkPolicy in the same pass

Alfredo Lopeze29b7be440ff
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index f5fc6d356..11360c072 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.403",
+ "version": "0.1.404",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.403",
+ "version": "0.1.404",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index ed7e96947..924727e21 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.403",
+ "version": "0.1.404",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 0b593d43e..2fd7bd604 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -217,13 +217,12 @@ type RunLaneNetworkPolicyApplyResult =
| {
policy: string
action: 'retry'
- reason:
- | 'terminating'
- | 'absent-after-write-conflict'
- | 'deleted-before-replace'
- | 'replace-conflicted-twice'
+ reason: 'terminating' | 'contended'
}
| { policy: string; action: 'conflict'; reason: NetworkPolicyConflictReason }
+// Read-then-write rounds per NetworkPolicy and pass. Three rounds is at most
+// three writes, the same as the create plus two replaces it replaced.
+const NETWORK_POLICY_APPLY_ROUNDS = 3
const MCP_HOST_READINESS_WAIT_TIMEOUT_MS = 4 * 60_000
const DNS_SUBDOMAIN_RE =
/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*$/
@@ -4849,10 +4848,19 @@ export class WorkflowReconciler {
* write is a full-object PUT: desired spec and annotations, plus the live
* labels, finalizers and annotations outside the owned key set. Owned
* annotations that desired no longer carries are therefore removed, and
- * foreign ones are kept. Ownership conflicts, terminating objects, a
- * policy absent on the re-read after a 409, a 404 on the PUT (reason
- * `deleted-before-replace`) and a replace that conflicts twice are returned,
- * not thrown: the workflow reconcile catch turns every error it does not
+ * foreign ones are kept.
+ *
+ * Each round reads the policy again. An absent policy is created; a present
+ * one goes through `decideNetworkPolicyConvergence`. A 409 on the create, or
+ * a 404 or 409 on the PUT, means another writer moved the object between
+ * the read and the write, so the next round reads it again. A policy that
+ * disappears is therefore recreated in the same pass. After
+ * `NETWORK_POLICY_APPLY_ROUNDS` rounds the policy is reported as
+ * `retry/contended`; that bound matches the write count of the previous
+ * create-then-replace-twice apply.
+ *
+ * Ownership conflicts, terminating objects and contention are returned, not
+ * thrown: the workflow reconcile catch turns every error it does not
* classify as transient into a terminal `failed`. Any other API error still
* propagates.
*/
@@ -4864,20 +4872,20 @@ export class WorkflowReconciler {
if (!name || !namespace)
throw new Error('NetworkPolicy metadata.name and namespace are required')
- let existing = await this.readNetworkPolicyOrNull(name, namespace)
- if (!existing) {
- try {
- await this.deps.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
- this.log.info(`Created NetworkPolicy "${name}"`)
- return { policy: name, action: 'created' }
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) throw error
+ for (let round = 0; round < NETWORK_POLICY_APPLY_ROUNDS; round += 1) {
+ const existing = await this.readNetworkPolicyOrNull(name, namespace)
+ if (!existing) {
+ try {
+ await this.deps.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
+ this.log.info(`Created NetworkPolicy "${name}"`)
+ return { policy: name, action: 'created' }
+ } catch (error: unknown) {
+ // Another writer created it after our read: read it again.
+ if (getErrorCode(error) !== 409) throw error
+ continue
+ }
}
- existing = await this.readNetworkPolicyOrNull(name, namespace)
- if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)
- }
- for (let attempt = 0; attempt < 2; attempt += 1) {
const decision: NetworkPolicyConvergenceDecision = decideNetworkPolicyConvergence(
'workload-egress',
policy,
@@ -4914,21 +4922,10 @@ export class WorkflowReconciler {
this.log.info(`Updated NetworkPolicy "${name}"`, { namespace, reason: decision.reason })
return { policy: name, action: 'replaced' }
} catch (error: unknown) {
+ // 404: deleted after our read. 409: changed after our read. Either
+ // way the next round reads it again and decides from that.
const code = getErrorCode(error)
- // Deleted between the read and the PUT: no conflict happened, so
- // this is its own retry reason.
- if (code === 404) return this.networkPolicyDeletedBeforeReplace(name, namespace)
- // A stale resourceVersion gets one re-read and re-decision.
- if (code !== 409) throw error
- if (attempt === 1) {
- // Contention, not a defect: another writer changed the policy again
- // between the re-read and the retry. Leave it for a later pass.
- this.log.warn(`NetworkPolicy "${name}" replace conflicted twice; retrying later`, {
- namespace,
- reason: 'replace-conflicted-twice',
- })
- return { policy: name, action: 'retry', reason: 'replace-conflicted-twice' }
- }
+ if (code !== 404 && code !== 409) throw error
}
break
default: {
@@ -4942,35 +4939,14 @@ export class WorkflowReconciler {
)
}
}
- existing = await this.readNetworkPolicyOrNull(name, namespace)
- if (!existing) return this.networkPolicyAbsentAfterConflict(name, namespace)
}
- throw new Error(`NetworkPolicy "${name}" apply loop exited without a result`)
- }
-
- // Both retries below leave the policy absent. The recipe reconciler's
- // in-progress and active short-circuits return before this apply, so the
- // message names the first pass that reaches it, not the next pass.
- private networkPolicyAbsentAfterConflict(
- name: string,
- namespace: string
- ): RunLaneNetworkPolicyApplyResult {
- this.log.warn(
- `NetworkPolicy "${name}" vanished after a write conflict; a later pass that reaches the apply recreates it`,
- { namespace, reason: 'absent-after-write-conflict' }
- )
- return { policy: name, action: 'retry', reason: 'absent-after-write-conflict' }
- }
-
- private networkPolicyDeletedBeforeReplace(
- name: string,
- namespace: string
- ): RunLaneNetworkPolicyApplyResult {
+ // Contention, not a defect: another writer moved the policy after every
+ // one of our reads. Leave it for a later pass.
this.log.warn(
- `NetworkPolicy "${name}" was deleted before the replace; a later pass that reaches the apply recreates it`,
- { namespace, reason: 'deleted-before-replace' }
+ `NetworkPolicy "${name}" is still contended after ${NETWORK_POLICY_APPLY_ROUNDS} write attempts; retrying later`,
+ { namespace, reason: 'contended' }
)
- return { policy: name, action: 'retry', reason: 'deleted-before-replace' }
+ return { policy: name, action: 'retry', reason: 'contended' }
}
private async safeDelete(deleteFn: () => Promise<unknown>): Promise<void> {
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index aa3d7ab9f..e0d4c23fc 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3833,7 +3833,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
}
})
- it('flags a retry, without failing, when a concurrent writer wins both replaces of a run-lane policy', async () => {
+ it('flags a contended retry, without failing, after a concurrent writer wins all three replaces of a run-lane policy', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
@@ -3849,7 +3849,7 @@ describe('WorkflowReconciler — reconcile loop', () => {
const liveResourceVersion = contended!.metadata!.resourceVersion
// Another writer updates the policy just before each of our replaces, so
- // both carry a resourceVersion that is already stale.
+ // every one carries a resourceVersion that is already stale.
const apiserverReplace = api.replaceNamespacedNetworkPolicy.getMockImplementation()!
let concurrentWrites = 0
api.replaceNamespacedNetworkPolicy.mockImplementation(async arg => {
@@ -3873,23 +3873,27 @@ describe('WorkflowReconciler — reconcile loop', () => {
api.readNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
- ).toHaveLength(2)
+ ).toHaveLength(3)
+ // Bounded: three writes, the same ceiling as the create-then-two-replaces
+ // apply this PR replaces.
expect(
api.replaceNamespacedNetworkPolicy.mock.calls
.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
.map(([arg]) => arg.body.metadata?.resourceVersion)
- ).toEqual([liveResourceVersion, 'concurrent-1'])
+ ).toEqual([liveResourceVersion, 'concurrent-1', 'concurrent-2'])
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBe(true)
expect(
- logs.entries.some(
- entry =>
- entry.level === 'warn' &&
- entry.reason === 'replace-conflicted-twice' &&
- String(entry.msg).includes('test-wf-coord-to-wrc')
+ logs.entries.filter(
+ entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
)
- ).toBe(true)
+ ).toEqual([
+ expect.objectContaining({
+ reason: 'contended',
+ msg: 'NetworkPolicy "test-wf-coord-to-wrc" is still contended after 3 write attempts; retrying later',
+ }),
+ ])
} finally {
logs.restore()
}
@@ -3899,8 +3903,10 @@ describe('WorkflowReconciler — reconcile loop', () => {
async function reconcileWithFailingReplace(
failReplace: (
arg: { name: string; namespace: string },
- live: Map<string, k8s.V1NetworkPolicy>
- ) => Promise<never> | undefined
+ live: Map<string, k8s.V1NetworkPolicy>,
+ api: ReturnType<typeof makeApiserverNetworkingApi>['api']
+ ) => Promise<never> | undefined,
+ logs?: { entries: Array<Record<string, unknown>> }
) {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
@@ -3912,55 +3918,66 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(first.workflowPhase).not.toBe('failed')
const drifted = apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
expect(drifted).toBeDefined()
+ // What the first pass stored: the desired policy as the apiserver keeps it.
+ const convergedSpec = structuredClone(drifted!.spec!)
drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
const apiserverReplace =
apiserver.api.replaceNamespacedNetworkPolicy.getMockImplementation()!
apiserver.api.replaceNamespacedNetworkPolicy.mockImplementation(
- async arg => failReplace(arg, apiserver.live) ?? apiserverReplace(arg)
+ async arg => failReplace(arg, apiserver.live, apiserver.api) ?? apiserverReplace(arg)
)
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
+ apiserver.api.createNamespacedNetworkPolicy.mockClear()
+ // Only the second pass is under test; drop the first pass's "Created" lines.
+ logs?.entries.splice(0)
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
const contendedCalls = (mock: typeof apiserver.api.readNamespacedNetworkPolicy) =>
mock.mock.calls.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
return {
second,
+ apiserver,
+ convergedSpec,
reads: contendedCalls(apiserver.api.readNamespacedNetworkPolicy),
replaces: apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
),
+ creates: apiserver.api.createNamespacedNetworkPolicy.mock.calls.filter(
+ ([arg]) => arg.body.metadata?.name === 'test-wf-coord-to-wrc'
+ ),
}
}
- it('flags a retry, without failing, when the policy is deleted between the read and the replace', async () => {
+ it('recreates the policy in the same pass when it is deleted between the read and the replace', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
- const { second, reads, replaces } = await reconcileWithFailingReplace((arg, live) => {
- if (arg.name !== 'test-wf-coord-to-wrc') return undefined
- // Another actor deletes the policy after our read, so the
- // apiserver double answers the PUT with 404.
- expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
- return undefined
- })
-
- expect(reads).toHaveLength(1)
+ const { second, apiserver, convergedSpec, reads, replaces, creates } =
+ await reconcileWithFailingReplace((arg, live) => {
+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined
+ // Another actor deletes the policy after our read, so the
+ // apiserver double answers the PUT with 404.
+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
+ return undefined
+ }, logs)
+
+ // Read, 404 on the PUT, read again (absent), create.
+ expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
+ expect(creates).toHaveLength(1)
+ // The recreated object is the desired policy, nothing wider.
+ expect(
+ apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
+ ).toEqual(convergedSpec)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
- expect(second.networkPolicyRetryPending).toBe(true)
- // No write conflict happened: the PUT itself answered 404.
+ expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
- logs.entries.filter(
- entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
- )
- ).toEqual([
- expect.objectContaining({
- reason: 'deleted-before-replace',
- msg: 'NetworkPolicy "test-wf-coord-to-wrc" was deleted before the replace; a later pass that reaches the apply recreates it',
- }),
- ])
+ logs.entries
+ .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
+ .map(entry => [entry.level, entry.msg])
+ ).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
} finally {
logs.restore()
}
@@ -3982,32 +3999,71 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(second.message).toContain('HTTP-Code: 422')
})
- it('flags a retry, without failing, when the policy is deleted after a replace conflict', async () => {
+ it('recreates the policy in the same pass when it is deleted after a replace conflict', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
- const { second, reads, replaces } = await reconcileWithFailingReplace((arg, live) => {
- if (arg.name !== 'test-wf-coord-to-wrc') return undefined
- // Another actor deletes the policy and our PUT loses the race, so
- // the re-read after the conflict finds nothing.
- expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
- return Promise.reject({ code: 409 })
- })
+ const { second, apiserver, convergedSpec, reads, replaces, creates } =
+ await reconcileWithFailingReplace((arg, live) => {
+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined
+ // Another actor deletes the policy and our PUT loses the race, so
+ // the re-read after the conflict finds nothing.
+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
+ return Promise.reject({ code: 409 })
+ }, logs)
expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
+ expect(creates).toHaveLength(1)
+ expect(
+ apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
+ ).toEqual(convergedSpec)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
- expect(second.networkPolicyRetryPending).toBe(true)
+ expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
- logs.entries.filter(
- entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
- )
- ).toEqual([
- expect.objectContaining({
- reason: 'absent-after-write-conflict',
- msg: 'NetworkPolicy "test-wf-coord-to-wrc" vanished after a write conflict; a later pass that reaches the apply recreates it',
- }),
- ])
+ logs.entries
+ .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
+ .map(entry => [entry.level, entry.msg])
+ ).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('leaves an equal copy alone when another writer recreates the policy between our 404 and our create', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ let createRaces = 0
+ const { second, apiserver, convergedSpec, reads, replaces, creates } =
+ await reconcileWithFailingReplace((arg, live, api) => {
+ if (arg.name !== 'test-wf-coord-to-wrc') return undefined
+ expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
+ // Before our create lands, another writer stores the desired
+ // policy, so our POST answers 409 and the next read finds it.
+ const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
+ api.createNamespacedNetworkPolicy.mockImplementationOnce(async create => {
+ createRaces += 1
+ await apiserverCreate(create)
+ return apiserverCreate(create)
+ })
+ return undefined
+ }, logs)
+
+ expect(createRaces).toBe(1)
+ // Read, 404 on the PUT, read (absent), create 409, read (equal).
+ expect(reads).toHaveLength(3)
+ expect(replaces).toHaveLength(1)
+ expect(creates).toHaveLength(1)
+ expect(
+ apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
+ ).toEqual(convergedSpec)
+ expect(second.workflowPhase).not.toBe('failed')
+ expect(second.networkPolicyRetryPending).toBeFalsy()
+ expect(
+ logs.entries
+ .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
+ .map(entry => [entry.level, entry.msg])
+ ).toEqual([['info', 'Unchanged NetworkPolicy "test-wf-coord-to-wrc"; skipping update']])
} finally {
logs.restore()
}
@@ -4171,13 +4227,50 @@ describe('WorkflowReconciler — reconcile loop', () => {
})
})
- it('flags a retry, without failing, when a create conflicts and the re-read finds nothing', async () => {
+ it('creates again in the same pass when a create conflicts and the re-read finds nothing', async () => {
+ const logs = captureRunLaneNetworkPolicyLogs()
+ try {
+ const { api, live, key } = makeApiserverNetworkingApi()
+ const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
+ let conflicts = 0
+ api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
+ // The winner of the first race is deleted again before our re-read.
+ if (arg.body.metadata?.name === CONTENDED && conflicts === 0) {
+ conflicts += 1
+ throw { code: 409 }
+ }
+ return apiserverCreate(arg)
+ })
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(conflicts).toBe(1)
+ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(2)
+ expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
+ expect(live.has(key('sandbox-recipes', CONTENDED))).toBe(true)
+ expect(result.phase).not.toBe('failed')
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(result.networkPolicyRetryPending).toBeFalsy()
+ expect(
+ logs.entries
+ .filter(entry => String(entry.msg).includes(CONTENDED))
+ .map(entry => [entry.level, entry.msg])
+ ).toEqual([['info', `Created NetworkPolicy "${CONTENDED}"`]])
+ } finally {
+ logs.restore()
+ }
+ })
+
+ it('flags a contended retry after three creates that all conflict, and writes no fourth time', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api } = makeApiserverNetworkingApi()
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
- // The winner of the race is deleted again before our re-read.
+ // Every winner is deleted again before our re-read.
if (arg.body.metadata?.name === CONTENDED) throw { code: 409 }
return apiserverCreate(arg)
})
@@ -4186,25 +4279,50 @@ describe('WorkflowReconciler — reconcile loop', () => {
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
- expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
- expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
+ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(3)
+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(3)
expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
expect(result.networkPolicyRetryPending).toBe(true)
expect(
- logs.entries.some(
- entry =>
- entry.level === 'warn' &&
- String(entry.msg).includes(CONTENDED) &&
- String(entry.msg).includes('vanished')
+ logs.entries.filter(
+ entry => entry.level === 'warn' && String(entry.msg).includes(CONTENDED)
)
- ).toBe(true)
+ ).toEqual([
+ expect.objectContaining({
+ reason: 'contended',
+ msg: `NetworkPolicy "${CONTENDED}" is still contended after 3 write attempts; retrying later`,
+ }),
+ ])
} finally {
logs.restore()
}
})
+ it('fails the run on a create error other than 409 and does not replace', async () => {
+ const { api } = makeApiserverNetworkingApi()
+ const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
+ api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
+ if (arg.body.metadata?.name === CONTENDED) {
+ throw Object.assign(new Error('HTTP-Code: 500 Message: etcd unavailable'), {
+ code: 500,
+ })
+ }
+ return apiserverCreate(arg)
+ })
+ const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
+ const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
+
+ const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
+
+ expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)
+ expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
+ expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
+ // A 500 is not a conflict: it leaves the apply and reaches the reconcile catch.
+ expect(result.message).toContain('HTTP-Code: 500')
+ })
+
it('fails the run on a read error other than 404 and writes nothing for that policy', async () => {
const { api } = makeApiserverNetworkingApi()
const apiserverRead = api.readNamespacedNetworkPolicy.getMockImplementation()!