Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): address #998 review findings (#760)

Alfredo Lopez5ce3cbb3faf3
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index a6da729fe..f91d0bc0b 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.465",
+ "version": "0.1.466",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.465",
+ "version": "0.1.466",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 5a07975d4..2aa529698 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.465",
+ "version": "0.1.466",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/k8sErrors.ts b/workflow-recipes/src/reconciler/k8sErrors.ts
index 1e2cb2add..6fb66fed1 100644
--- a/workflow-recipes/src/reconciler/k8sErrors.ts
+++ b/workflow-recipes/src/reconciler/k8sErrors.ts
@@ -196,13 +196,20 @@ export class RetryableReconcileError extends Error {
}
/**
- * A read-first apply read 404, its POST got a 409, and the re-read of the
- * object that won the race is a 404 again. There is nothing to replace and no
- * live object to judge, so the pass stops and asks for a fresh reconciliation
+ * A write got a 409 and the re-read of the conflicting object is a 404. After
+ * a create conflict (`after: 'create'`, the default) a read-first apply read
+ * 404, its POST got a 409, and the object that won the race is gone again.
+ * After a replace conflict (`after: 'replace'`) the object read before the
+ * PUT was deleted before the retry could read it. Either way there is no live
+ * object to judge, so the pass stops and asks for a fresh reconciliation
* instead of guessing with a second POST or a PUT that can only 404.
*/
export class ResourceVanishedAfterConflictError extends RetryableReconcileError {
- constructor(label: string, options?: { cause?: unknown }) {
- super(`${label} disappeared after create conflict; a fresh reconciliation is required`, options)
+ constructor(label: string, options?: { cause?: unknown; after?: 'create' | 'replace' }) {
+ const after = options?.after ?? 'create'
+ super(
+ `${label} disappeared after ${after} conflict; a fresh reconciliation is required`,
+ options && 'cause' in options ? { cause: options.cause } : undefined
+ )
}
}
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
index e005a844a..1360e483b 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
@@ -1551,6 +1551,8 @@ describe('ensureRecipeContext', () => {
)
expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()
+ // Liveness witness: the writer read the Context before deciding not to write.
+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()
})
@@ -1709,6 +1711,8 @@ describe('ensureRecipeContext', () => {
)
expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()
+ // Liveness witness: the writer read the Context before deciding not to write.
+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()
})
@@ -1867,6 +1871,8 @@ describe('ensureRecipeContext', () => {
)
expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()
+ // Liveness witness: the writer read the Context before deciding not to write.
+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()
})
@@ -1942,8 +1948,10 @@ describe('read-first Context apply (#760)', () => {
/**
* Stateful Context double: GET answers 404 while no Context is stored, a
- * create of a stored Context answers 409, a replace stores the body and bumps
- * resourceVersion. Every other plural reads as absent and accepts writes.
+ * create of a stored Context answers 409, a replace answers 404 without a
+ * stored Context and 409 on a stale resourceVersion, and otherwise stores the
+ * body and bumps resourceVersion. Every other plural reads as absent and
+ * accepts writes.
*/
function contextStore(initial: LiveContext | null) {
let live = initial ? structuredClone(initial) : null
@@ -1967,10 +1975,12 @@ describe('read-first Context apply (#760)', () => {
.fn()
.mockImplementation((args: { plural: string; body: LiveContext }) => {
if (args.plural !== 'contexts') return Promise.resolve({})
- const rv = Number((live?.metadata?.resourceVersion as string | undefined) ?? '0')
+ if (!live) return Promise.reject({ code: 404 })
+ const rv = live.metadata?.resourceVersion as string | undefined
+ if (args.body.metadata?.resourceVersion !== rv) return Promise.reject({ code: 409 })
live = {
...structuredClone(args.body),
- metadata: { ...args.body.metadata, resourceVersion: String(rv + 1) },
+ metadata: { ...args.body.metadata, resourceVersion: String(Number(rv) + 1) },
}
return Promise.resolve({})
}),
@@ -1986,10 +1996,10 @@ describe('read-first Context apply (#760)', () => {
return { customApi, deps, store: (next: LiveContext | null) => (live = next) }
}
- const contextCalls = (fn: ReturnType<typeof vi.fn>, name = CONTEXT_NAME) =>
+ const contextCalls = (fn: ReturnType<typeof vi.fn>) =>
fn.mock.calls.filter(([args]) => {
const a = args as { plural: string; name?: string; body?: { metadata?: { name?: string } } }
- return a.plural === 'contexts' && (a.name ?? a.body?.metadata?.name) === name
+ return a.plural === 'contexts' && (a.name ?? a.body?.metadata?.name) === CONTEXT_NAME
}).length
const loggedFor = (level: ReturnType<typeof vi.fn>, message: string) =>
level.mock.calls.filter(
@@ -2109,18 +2119,39 @@ describe('read-first Context apply (#760)', () => {
// The conflicting object is gone again by the time we re-read it.
customApi.createNamespacedCustomObject.mockRejectedValueOnce({ code: 409 })
- await expect(ensure(deps)).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)
+ const outcome = ensure(deps)
+ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)
+ await expect(outcome).rejects.toThrow('disappeared after create conflict')
expect(contextCalls(customApi.getNamespacedCustomObject)).toBe(2)
expect(contextCalls(customApi.replaceNamespacedCustomObject)).toBe(0)
})
+
+ it('C8: a Context that vanishes after a replace conflict asks for a fresh reconcile', async () => {
+ const existing = unchangedPrivateContext()
+ existing.spec = { contextId: CONTEXT_NAME, mcpServers: ['server-a'] }
+ const { customApi, deps, store } = contextStore(existing)
+ // Another writer bumps the Context, then it is deleted before our retry reads it.
+ customApi.replaceNamespacedCustomObject.mockImplementationOnce(() => {
+ store(null)
+ return Promise.reject({ code: 409 })
+ })
+
+ const outcome = ensure(deps)
+ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)
+ await expect(outcome).rejects.toThrow('disappeared after replace conflict')
+
+ expect(contextCalls(customApi.getNamespacedCustomObject)).toBe(2)
+ expect(contextCalls(customApi.replaceNamespacedCustomObject)).toBe(1)
+ expect(contextCalls(customApi.createNamespacedCustomObject)).toBe(0)
+ })
})
describe('delegateTransportWorkloads', () => {
it('returns delegated server names and creates per-recipe Context (H-04)', async () => {
const mockCustomApi = {
- // Call 1: McpServer pre-check GET → 404 (new server)
- // Call 2: ensureRecipeContext createNamespacedCustomObject → succeeds
+ // GET 1: McpServer pre-check → 404 (new server), then McpServer create.
+ // GET 2: ensureRecipeContext reads the Context → 404, then Context create.
createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
getNamespacedCustomObject: contextAbsentGet(
vi
@@ -2145,6 +2176,20 @@ describe('delegateTransportWorkloads', () => {
expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(1)
// createNamespacedCustomObject called twice: McpServer + per-recipe Context
expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledTimes(2)
+ // The Context was read before it was created (#760).
+ const contextGet =
+ mockCustomApi.getNamespacedCustomObject.mock.invocationCallOrder[
+ mockCustomApi.getNamespacedCustomObject.mock.calls.findIndex(
+ ([args]) => (args as { plural: string }).plural === 'contexts'
+ )
+ ]
+ const contextCreate =
+ mockCustomApi.createNamespacedCustomObject.mock.invocationCallOrder[
+ mockCustomApi.createNamespacedCustomObject.mock.calls.findIndex(
+ ([args]) => (args as { plural: string }).plural === 'contexts'
+ )
+ ]
+ expect(contextGet).toBeLessThan(contextCreate)
// Verify the Context CRD was created (second call), NOT the shared allowlist patched
expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(
expect.objectContaining({
@@ -2423,8 +2468,14 @@ describe('delegateTransportWorkloads', () => {
coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],
}
- await delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())
+ await expect(
+ delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())
+ ).resolves.toEqual(['test-recipe-redis-mcp'])
+ // Liveness witness: the per-recipe Context was written by create.
+ expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(
+ expect.objectContaining({ plural: 'contexts' })
+ )
// patchNamespacedCustomObject must NOT be called — no shared context mutation
expect(mockCustomApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
})
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.ts
index 93e2248d7..ee529d33b 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts
@@ -984,13 +984,14 @@ export async function ensureRecipeContext(
// Every read in here runs after a conflict (create or replace), so the
// object is known to have existed; a 404 asks for a fresh reconciliation.
- const rereadContext = async (): Promise<ExistingContextSnapshot> => {
+ const rereadContext = async (after: 'create' | 'replace'): Promise<ExistingContextSnapshot> => {
try {
return await readContext()
} catch (error) {
if (getErrorCode(error) === 404) {
throw new ResourceVanishedAfterConflictError(`Context "${contextName}" in ${namespace}`, {
cause: error,
+ after,
})
}
throw error
@@ -1003,7 +1004,12 @@ export async function ensureRecipeContext(
initial?: ExistingContextSnapshot
): Promise<{ wrote: boolean }> => {
for (let attempt = 1; attempt <= CONTEXT_REPLACE_CONFLICT_RETRIES; attempt += 1) {
- const existing = attempt === 1 && initial ? initial : await rereadContext()
+ // Without a snapshot, attempt 1 follows a create conflict; every later
+ // attempt follows a replace conflict.
+ const existing =
+ attempt === 1 && initial
+ ? initial
+ : await rereadContext(attempt === 1 ? 'create' : 'replace')
// Semantic post-merge equality. Do not stamp clerum.io/spec-hash on a
// shared Context: N recipe writers would flap the annotation the same
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 0900d86db..b1c04cdaf 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -248,10 +248,11 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R
* succeeds.
*
* `patch`, when given, merges `metadata.annotations` and the top-level `spec`
- * keys of the body into the stored object and bumps its resourceVersion. It is
- * not a strategic merge: assertions about what a patch sent read the request
- * body, not the stored object. A patch to an object the store never held
- * follows `unseenReplace`.
+ * keys of the body into the stored object, bumps its resourceVersion, and bumps
+ * its generation when the body carries `spec`, as the apiserver does. It is not
+ * a strategic merge: assertions about what a patch sent read the request body,
+ * not the stored object. A patch to an object the store does not hold answers
+ * 404, as the apiserver does, whatever `unseenReplace` says.
*/
function installLiveStore(
api: {
@@ -327,13 +328,11 @@ function installLiveStore(
}) => {
const k = key(namespace, name)
const current = live.get(k)
- if (!current) {
- if (unseenReplace === 'reject') throw { code: 404 }
- return {}
- }
+ if (!current) throw { code: 404 }
const metadata = current.metadata as LiveManifest['metadata'] & {
annotations?: Record<string, string>
resourceVersion?: string
+ generation?: number
}
live.set(k, {
...current,
@@ -341,6 +340,7 @@ function installLiveStore(
...metadata,
annotations: { ...metadata.annotations, ...body.metadata?.annotations },
resourceVersion: String(Number(metadata.resourceVersion ?? '1') + 1),
+ generation: (metadata.generation ?? 1) + (body.spec ? 1 : 0),
},
spec: { ...(current.spec as Record<string, unknown>), ...body.spec },
})
@@ -4071,8 +4071,10 @@ describe('WorkflowRecipeReconciler', () => {
})
// #760: the StatefulSet writer reads before it creates, so a pass over an
- // object that already exists writes nothing. Every count below is filtered
- // by name: the same read serves the readiness probe and the legacy checks.
+ // object that already exists writes nothing. Every read and create count
+ // below is filtered by name, because the same read serves the readiness
+ // probe and the legacy checks. Patch counts stay unfiltered: nothing else
+ // patches a StatefulSet, so the unfiltered zero is the stricter assertion.
describe('read-first StatefulSet apply (#760)', () => {
type StsBody = {
metadata: { name: string; annotations: Record<string, string> }
@@ -4197,7 +4199,10 @@ describe('WorkflowRecipeReconciler', () => {
await expect(ensure()).rejects.toBe(failure)
// Witness: the failing read happened; the create (which would succeed) did not.
- expect(read).toHaveBeenCalledTimes(1)
+ const writerReads = read.mock.calls.filter(
+ ([args]) => args.namespace === namespace && String(args.name).includes(workload.id)
+ )
+ expect(writerReads).toHaveLength(1)
expect(create).toHaveBeenCalledTimes(0)
expect(patch).toHaveBeenCalledTimes(0)
})
@@ -4250,6 +4255,9 @@ describe('WorkflowRecipeReconciler', () => {
).ensureStatefulSet(workload, recipe, 'minimal', {})
expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({
@@ -4306,11 +4314,17 @@ describe('WorkflowRecipeReconciler', () => {
const existing = clone(createdBody!)
existing.metadata.annotations = { [SPEC_HASH]: 'stale-or-missing' }
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
mockAppsApi.replaceNamespacedStatefulSet.mockClear()
mockAppsApi.patchNamespacedStatefulSet.mockClear()
+ // The existing object lives in the read mock above, not in the live store,
+ // so the patch answers here instead of the store's 404 for an absent object.
+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
await (
reconciler as unknown as {
@@ -4348,11 +4362,17 @@ describe('WorkflowRecipeReconciler', () => {
const existing = clone(existingBody!)
existing.metadata.annotations = { [SPEC_HASH]: 'stale-old-template' }
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
mockAppsApi.replaceNamespacedStatefulSet.mockClear()
mockAppsApi.patchNamespacedStatefulSet.mockClear()
+ // The existing object lives in the read mock above, not in the live store,
+ // so the patch answers here instead of the store's 404 for an absent object.
+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
await (
reconciler as unknown as {
@@ -4416,11 +4436,17 @@ describe('WorkflowRecipeReconciler', () => {
},
status: { phase: 'Pending' },
}))
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
mockAppsApi.replaceNamespacedStatefulSet.mockClear()
mockAppsApi.patchNamespacedStatefulSet.mockClear()
+ // The existing object lives in the read mock above, not in the live store,
+ // so the patch answers here instead of the store's 404 for an absent object.
+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
await (
reconciler as unknown as {
@@ -4513,11 +4539,17 @@ describe('WorkflowRecipeReconciler', () => {
},
status: { phase: 'Pending' },
}))
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
mockAppsApi.replaceNamespacedStatefulSet.mockClear()
mockAppsApi.patchNamespacedStatefulSet.mockClear()
+ // The existing object lives in the read mock above, not in the live store,
+ // so the patch answers here instead of the store's 404 for an absent object.
+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
await (
reconciler as unknown as {
@@ -4583,6 +4615,9 @@ describe('WorkflowRecipeReconciler', () => {
const existing = clone(existingBody!)
existing.metadata.annotations = { [SPEC_HASH]: 'stale-storage' }
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
@@ -4629,6 +4664,9 @@ describe('WorkflowRecipeReconciler', () => {
const existing = clone(createdBody!)
existing.metadata.annotations = { [SPEC_HASH]: 'stale-immutable-drift' }
existing.spec.serviceName = 'manually-mutated-headless-service'
+ // Not reached on the read-first path (#760). If the writer went back to
+ // POST-first, this 409 would keep the old path working and only the
+ // zero-create assertion below would catch it.
mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })
mockAppsApi.createNamespacedStatefulSet.mockClear()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
index 355f4cc9a..049796f67 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
@@ -894,6 +894,7 @@ describe('Workflow Reconciler Bifurcation', () => {
expect(mockCoreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()
expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()
+ expect(mockAppsApi.patchNamespacedStatefulSet).not.toHaveBeenCalled()
expect(mockCoreApi.createNamespacedService).not.toHaveBeenCalled()
expect(mockCoreApi.replaceNamespacedService).not.toHaveBeenCalled()
expect(mockCoreApi.createNamespacedConfigMap).not.toHaveBeenCalled()