Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): address #998 review findings (#760)
Alfredo Lopez5ce3cbb3faf3
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex a6da729fe..f91d0bc0b 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.465",+ "version": "0.1.466","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.465",+ "version": "0.1.466","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 5a07975d4..2aa529698 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.465",+ "version": "0.1.466","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/k8sErrors.ts b/workflow-recipes/src/reconciler/k8sErrors.tsindex 1e2cb2add..6fb66fed1 100644--- a/workflow-recipes/src/reconciler/k8sErrors.ts+++ b/workflow-recipes/src/reconciler/k8sErrors.ts@@ -196,13 +196,20 @@ export class RetryableReconcileError extends Error {}/**- * A read-first apply read 404, its POST got a 409, and the re-read of the- * object that won the race is a 404 again. There is nothing to replace and no- * live object to judge, so the pass stops and asks for a fresh reconciliation+ * A write got a 409 and the re-read of the conflicting object is a 404. After+ * a create conflict (`after: 'create'`, the default) a read-first apply read+ * 404, its POST got a 409, and the object that won the race is gone again.+ * After a replace conflict (`after: 'replace'`) the object read before the+ * PUT was deleted before the retry could read it. Either way there is no live+ * object to judge, so the pass stops and asks for a fresh reconciliation* instead of guessing with a second POST or a PUT that can only 404.*/export class ResourceVanishedAfterConflictError extends RetryableReconcileError {- constructor(label: string, options?: { cause?: unknown }) {- super(`${label} disappeared after create conflict; a fresh reconciliation is required`, options)+ constructor(label: string, options?: { cause?: unknown; after?: 'create' | 'replace' }) {+ const after = options?.after ?? 'create'+ super(+ `${label} disappeared after ${after} conflict; a fresh reconciliation is required`,+ options && 'cause' in options ? { cause: options.cause } : undefined+ )}}diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex e005a844a..1360e483b 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -1551,6 +1551,8 @@ describe('ensureRecipeContext', () => {)expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()+ // Liveness witness: the writer read the Context before deciding not to write.+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()})@@ -1709,6 +1711,8 @@ describe('ensureRecipeContext', () => {)expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()+ // Liveness witness: the writer read the Context before deciding not to write.+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()})@@ -1867,6 +1871,8 @@ describe('ensureRecipeContext', () => {)expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()+ // Liveness witness: the writer read the Context before deciding not to write.+ expect(mockCustomApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()})@@ -1942,8 +1948,10 @@ describe('read-first Context apply (#760)', () => {/*** Stateful Context double: GET answers 404 while no Context is stored, a- * create of a stored Context answers 409, a replace stores the body and bumps- * resourceVersion. Every other plural reads as absent and accepts writes.+ * create of a stored Context answers 409, a replace answers 404 without a+ * stored Context and 409 on a stale resourceVersion, and otherwise stores the+ * body and bumps resourceVersion. Every other plural reads as absent and+ * accepts writes.*/function contextStore(initial: LiveContext | null) {let live = initial ? structuredClone(initial) : null@@ -1967,10 +1975,12 @@ describe('read-first Context apply (#760)', () => {.fn().mockImplementation((args: { plural: string; body: LiveContext }) => {if (args.plural !== 'contexts') return Promise.resolve({})- const rv = Number((live?.metadata?.resourceVersion as string | undefined) ?? '0')+ if (!live) return Promise.reject({ code: 404 })+ const rv = live.metadata?.resourceVersion as string | undefined+ if (args.body.metadata?.resourceVersion !== rv) return Promise.reject({ code: 409 })live = {...structuredClone(args.body),- metadata: { ...args.body.metadata, resourceVersion: String(rv + 1) },+ metadata: { ...args.body.metadata, resourceVersion: String(Number(rv) + 1) },}return Promise.resolve({})}),@@ -1986,10 +1996,10 @@ describe('read-first Context apply (#760)', () => {return { customApi, deps, store: (next: LiveContext | null) => (live = next) }}- const contextCalls = (fn: ReturnType<typeof vi.fn>, name = CONTEXT_NAME) =>+ const contextCalls = (fn: ReturnType<typeof vi.fn>) =>fn.mock.calls.filter(([args]) => {const a = args as { plural: string; name?: string; body?: { metadata?: { name?: string } } }- return a.plural === 'contexts' && (a.name ?? a.body?.metadata?.name) === name+ return a.plural === 'contexts' && (a.name ?? a.body?.metadata?.name) === CONTEXT_NAME}).lengthconst loggedFor = (level: ReturnType<typeof vi.fn>, message: string) =>level.mock.calls.filter(@@ -2109,18 +2119,39 @@ describe('read-first Context apply (#760)', () => {// The conflicting object is gone again by the time we re-read it.customApi.createNamespacedCustomObject.mockRejectedValueOnce({ code: 409 })- await expect(ensure(deps)).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)+ const outcome = ensure(deps)+ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)+ await expect(outcome).rejects.toThrow('disappeared after create conflict')expect(contextCalls(customApi.getNamespacedCustomObject)).toBe(2)expect(contextCalls(customApi.replaceNamespacedCustomObject)).toBe(0)})++ it('C8: a Context that vanishes after a replace conflict asks for a fresh reconcile', async () => {+ const existing = unchangedPrivateContext()+ existing.spec = { contextId: CONTEXT_NAME, mcpServers: ['server-a'] }+ const { customApi, deps, store } = contextStore(existing)+ // Another writer bumps the Context, then it is deleted before our retry reads it.+ customApi.replaceNamespacedCustomObject.mockImplementationOnce(() => {+ store(null)+ return Promise.reject({ code: 409 })+ })++ const outcome = ensure(deps)+ await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)+ await expect(outcome).rejects.toThrow('disappeared after replace conflict')++ expect(contextCalls(customApi.getNamespacedCustomObject)).toBe(2)+ expect(contextCalls(customApi.replaceNamespacedCustomObject)).toBe(1)+ expect(contextCalls(customApi.createNamespacedCustomObject)).toBe(0)+ })})describe('delegateTransportWorkloads', () => {it('returns delegated server names and creates per-recipe Context (H-04)', async () => {const mockCustomApi = {- // Call 1: McpServer pre-check GET → 404 (new server)- // Call 2: ensureRecipeContext createNamespacedCustomObject → succeeds+ // GET 1: McpServer pre-check → 404 (new server), then McpServer create.+ // GET 2: ensureRecipeContext reads the Context → 404, then Context create.createNamespacedCustomObject: vi.fn().mockResolvedValue({}),getNamespacedCustomObject: contextAbsentGet(vi@@ -2145,6 +2176,20 @@ describe('delegateTransportWorkloads', () => {expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(1)// createNamespacedCustomObject called twice: McpServer + per-recipe Contextexpect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledTimes(2)+ // The Context was read before it was created (#760).+ const contextGet =+ mockCustomApi.getNamespacedCustomObject.mock.invocationCallOrder[+ mockCustomApi.getNamespacedCustomObject.mock.calls.findIndex(+ ([args]) => (args as { plural: string }).plural === 'contexts'+ )+ ]+ const contextCreate =+ mockCustomApi.createNamespacedCustomObject.mock.invocationCallOrder[+ mockCustomApi.createNamespacedCustomObject.mock.calls.findIndex(+ ([args]) => (args as { plural: string }).plural === 'contexts'+ )+ ]+ expect(contextGet).toBeLessThan(contextCreate)// Verify the Context CRD was created (second call), NOT the shared allowlist patchedexpect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(expect.objectContaining({@@ -2423,8 +2468,14 @@ describe('delegateTransportWorkloads', () => {coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],}- await delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())+ await expect(+ delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())+ ).resolves.toEqual(['test-recipe-redis-mcp'])+ // Liveness witness: the per-recipe Context was written by create.+ expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({ plural: 'contexts' })+ )// patchNamespacedCustomObject must NOT be called — no shared context mutationexpect(mockCustomApi.patchNamespacedCustomObject).not.toHaveBeenCalled()})diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex 93e2248d7..ee529d33b 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -984,13 +984,14 @@ export async function ensureRecipeContext(// Every read in here runs after a conflict (create or replace), so the// object is known to have existed; a 404 asks for a fresh reconciliation.- const rereadContext = async (): Promise<ExistingContextSnapshot> => {+ const rereadContext = async (after: 'create' | 'replace'): Promise<ExistingContextSnapshot> => {try {return await readContext()} catch (error) {if (getErrorCode(error) === 404) {throw new ResourceVanishedAfterConflictError(`Context "${contextName}" in ${namespace}`, {cause: error,+ after,})}throw error@@ -1003,7 +1004,12 @@ export async function ensureRecipeContext(initial?: ExistingContextSnapshot): Promise<{ wrote: boolean }> => {for (let attempt = 1; attempt <= CONTEXT_REPLACE_CONFLICT_RETRIES; attempt += 1) {- const existing = attempt === 1 && initial ? initial : await rereadContext()+ // Without a snapshot, attempt 1 follows a create conflict; every later+ // attempt follows a replace conflict.+ const existing =+ attempt === 1 && initial+ ? initial+ : await rereadContext(attempt === 1 ? 'create' : 'replace')// Semantic post-merge equality. Do not stamp clerum.io/spec-hash on a// shared Context: N recipe writers would flap the annotation the samediff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 0900d86db..b1c04cdaf 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -248,10 +248,11 @@ type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & R* succeeds.** `patch`, when given, merges `metadata.annotations` and the top-level `spec`- * keys of the body into the stored object and bumps its resourceVersion. It is- * not a strategic merge: assertions about what a patch sent read the request- * body, not the stored object. A patch to an object the store never held- * follows `unseenReplace`.+ * keys of the body into the stored object, bumps its resourceVersion, and bumps+ * its generation when the body carries `spec`, as the apiserver does. It is not+ * a strategic merge: assertions about what a patch sent read the request body,+ * not the stored object. A patch to an object the store does not hold answers+ * 404, as the apiserver does, whatever `unseenReplace` says.*/function installLiveStore(api: {@@ -327,13 +328,11 @@ function installLiveStore(}) => {const k = key(namespace, name)const current = live.get(k)- if (!current) {- if (unseenReplace === 'reject') throw { code: 404 }- return {}- }+ if (!current) throw { code: 404 }const metadata = current.metadata as LiveManifest['metadata'] & {annotations?: Record<string, string>resourceVersion?: string+ generation?: number}live.set(k, {...current,@@ -341,6 +340,7 @@ function installLiveStore(...metadata,annotations: { ...metadata.annotations, ...body.metadata?.annotations },resourceVersion: String(Number(metadata.resourceVersion ?? '1') + 1),+ generation: (metadata.generation ?? 1) + (body.spec ? 1 : 0),},spec: { ...(current.spec as Record<string, unknown>), ...body.spec },})@@ -4071,8 +4071,10 @@ describe('WorkflowRecipeReconciler', () => {})// #760: the StatefulSet writer reads before it creates, so a pass over an- // object that already exists writes nothing. Every count below is filtered- // by name: the same read serves the readiness probe and the legacy checks.+ // object that already exists writes nothing. Every read and create count+ // below is filtered by name, because the same read serves the readiness+ // probe and the legacy checks. Patch counts stay unfiltered: nothing else+ // patches a StatefulSet, so the unfiltered zero is the stricter assertion.describe('read-first StatefulSet apply (#760)', () => {type StsBody = {metadata: { name: string; annotations: Record<string, string> }@@ -4197,7 +4199,10 @@ describe('WorkflowRecipeReconciler', () => {await expect(ensure()).rejects.toBe(failure)// Witness: the failing read happened; the create (which would succeed) did not.- expect(read).toHaveBeenCalledTimes(1)+ const writerReads = read.mock.calls.filter(+ ([args]) => args.namespace === namespace && String(args.name).includes(workload.id)+ )+ expect(writerReads).toHaveLength(1)expect(create).toHaveBeenCalledTimes(0)expect(patch).toHaveBeenCalledTimes(0)})@@ -4250,6 +4255,9 @@ describe('WorkflowRecipeReconciler', () => {).ensureStatefulSet(workload, recipe, 'minimal', {})expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({@@ -4306,11 +4314,17 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(createdBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-or-missing' }+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()+ // The existing object lives in the read mock above, not in the live store,+ // so the patch answers here instead of the store's 404 for an absent object.+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})await (reconciler as unknown as {@@ -4348,11 +4362,17 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(existingBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-old-template' }+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()+ // The existing object lives in the read mock above, not in the live store,+ // so the patch answers here instead of the store's 404 for an absent object.+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})await (reconciler as unknown as {@@ -4416,11 +4436,17 @@ describe('WorkflowRecipeReconciler', () => {},status: { phase: 'Pending' },}))+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()+ // The existing object lives in the read mock above, not in the live store,+ // so the patch answers here instead of the store's 404 for an absent object.+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})await (reconciler as unknown as {@@ -4513,11 +4539,17 @@ describe('WorkflowRecipeReconciler', () => {},status: { phase: 'Pending' },}))+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)mockAppsApi.replaceNamespacedStatefulSet.mockClear()mockAppsApi.patchNamespacedStatefulSet.mockClear()+ // The existing object lives in the read mock above, not in the live store,+ // so the patch answers here instead of the store's 404 for an absent object.+ mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})await (reconciler as unknown as {@@ -4583,6 +4615,9 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(existingBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-storage' }+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)@@ -4629,6 +4664,9 @@ describe('WorkflowRecipeReconciler', () => {const existing = clone(createdBody!)existing.metadata.annotations = { [SPEC_HASH]: 'stale-immutable-drift' }existing.spec.serviceName = 'manually-mutated-headless-service'+ // Not reached on the read-first path (#760). If the writer went back to+ // POST-first, this 409 would keep the old path working and only the+ // zero-create assertion below would catch it.mockAppsApi.createNamespacedStatefulSet.mockRejectedValue({ code: 409 })mockAppsApi.createNamespacedStatefulSet.mockClear()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue(existing)diff --git a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.tsindex 355f4cc9a..049796f67 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts@@ -894,6 +894,7 @@ describe('Workflow Reconciler Bifurcation', () => {expect(mockCoreApi.replaceNamespacedSecret).not.toHaveBeenCalled()expect(mockAppsApi.createNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockAppsApi.replaceNamespacedStatefulSet).not.toHaveBeenCalled()+ expect(mockAppsApi.patchNamespacedStatefulSet).not.toHaveBeenCalled()expect(mockCoreApi.createNamespacedService).not.toHaveBeenCalled()expect(mockCoreApi.replaceNamespacedService).not.toHaveBeenCalled()expect(mockCoreApi.createNamespacedConfigMap).not.toHaveBeenCalled()