Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): keep pre-deploy vanish errors retryable (#760)
Alfredo Lopezfcc914a82b21
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex f91d0bc0b..8b72a97bb 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.466",+ "version": "0.1.467","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.466",+ "version": "0.1.467","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 2aa529698..6b07cc1bb 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.466",+ "version": "0.1.467","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex 1360e483b..21a1d6453 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -6,7 +6,7 @@import { describe, expect, it, vi } from 'vitest'import { WorkflowRecipeCRD } from '../types'import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'-import { ResourceVanishedAfterConflictError } from './k8sErrors'+import { ResourceVanishedAfterConflictError, RetryableReconcileError } from './k8sErrors'import {DelegationDeps,PRE_DEPLOY_ANNOTATION,@@ -1142,7 +1142,7 @@ describe('ensureRecipeContext', () => {expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()})- it('H04b — writes on 409 when live Context has no mcpServers (empty→filled is not a skip)', async () => {+ it('H04b — replaces an existing Context that has no mcpServers (empty→filled is not a skip)', async () => {const mockCustomApi = {createNamespacedCustomObject: vi.fn(),getNamespacedCustomObject: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '5' } }),@@ -3029,9 +3029,10 @@ describe('preDeployMcpServers', () => {})const recipe = makeRecipe()- await expect(preDeployMcpServers(deps, recipe, 'mcp-server', new Map())).rejects.toThrow(- 'Pre-deploy failed for workload(s): redis-mcp'- )+ const outcome = preDeployMcpServers(deps, recipe, 'mcp-server', new Map())+ await expect(outcome).rejects.toThrow('Pre-deploy failed for workload(s): redis-mcp')+ // A permanent failure stays terminal (#998 audit).+ await expect(outcome).rejects.not.toBeInstanceOf(RetryableReconcileError)})it('throws when the pre-deploy Context allowlist cannot be persisted', async () => {@@ -3053,9 +3054,123 @@ describe('preDeployMcpServers', () => {})const recipe = makeRecipe()- await expect(preDeployMcpServers(deps, recipe, 'mcp-server', new Map())).rejects.toThrow(- 'Pre-deploy Context allowlist failed'+ const outcome = preDeployMcpServers(deps, recipe, 'mcp-server', new Map())+ await expect(outcome).rejects.toThrow('Pre-deploy Context allowlist failed')+ // A permanent Context error stays terminal (#998 audit).+ await expect(outcome).rejects.not.toBeInstanceOf(RetryableReconcileError)+ })++ it('keeps a Context that vanished after a create conflict retryable on the pre-deploy path', async () => {+ // GET 404, POST 409, re-read 404: the writer raises ResourceVanishedAfterConflictError.+ // The pre-deploy wrapper must not turn it into a terminal error (#998 audit).+ const deps = makePreDeployDeps({+ customApi: {+ createNamespacedCustomObject: vi+ .fn()+ .mockImplementation((opts: { body: { kind?: string } }) =>+ opts.body.kind === 'Context' ? Promise.reject({ code: 409 }) : Promise.resolve({})+ ),+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),+ patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),+ } as any,+ })+ const recipe = makeRecipe()++ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(+ () => undefined,+ (e: unknown) => e+ )++ expect(error).toBeInstanceOf(RetryableReconcileError)+ expect((error as Error).message).toContain('Pre-deploy Context allowlist failed')+ expect((error as Error).message).toContain('disappeared after create conflict')+ expect((error as { cause?: unknown }).cause).toBeInstanceOf(ResourceVanishedAfterConflictError)+ })++ it('keeps a pre-deploy failure retryable when every workload failed retryably', async () => {+ // Service GET 404, POST 409, re-read 404 for the only transport workload.+ const deps = makePreDeployDeps({+ coreApi: {+ createNamespacedService: vi.fn().mockRejectedValue({ code: 409 }),+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ replaceNamespacedService: vi.fn().mockResolvedValue({}),+ deleteNamespacedService: vi.fn().mockResolvedValue({}),+ } as any,+ })+ const recipe = makeRecipe()++ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(+ () => undefined,+ (e: unknown) => e+ )++ expect(error).toBeInstanceOf(RetryableReconcileError)+ expect((error as Error).message).toContain('Pre-deploy failed for workload(s): redis-mcp')+ })++ it('keeps a pre-deploy failure terminal when any workload failed permanently', async () => {+ // redis-mcp: its Service vanishes after a create conflict (retryable).+ // files-mcp: its McpServer create fails with a permanent error.+ const deps = makePreDeployDeps({+ customApi: {+ createNamespacedCustomObject: vi+ .fn()+ .mockImplementation((opts: { body: { kind?: string; metadata?: { name?: string } } }) =>+ opts.body.kind === 'McpServer' && String(opts.body.metadata?.name).includes('files-mcp')+ ? Promise.reject(new Error('quota exceeded'))+ : Promise.resolve({})+ ),+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),+ patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),+ } as any,+ coreApi: {+ createNamespacedService: vi+ .fn()+ .mockImplementation((opts: { body: { metadata?: { name?: string } } }) =>+ String(opts.body.metadata?.name).includes('redis-mcp')+ ? Promise.reject({ code: 409 })+ : Promise.resolve({})+ ),+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ replaceNamespacedService: vi.fn().mockResolvedValue({}),+ deleteNamespacedService: vi.fn().mockResolvedValue({}),+ } as any,+ })+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'redis-mcp',+ type: 'deployment',+ image: 'clerum/redis-mcp:latest',+ port: 3000,+ transport: { type: 'streamableHttp', path: '/mcp' },+ },+ {+ id: 'files-mcp',+ type: 'deployment',+ image: 'clerum/files-mcp:latest',+ port: 3000,+ transport: { type: 'streamableHttp', path: '/mcp' },+ },+ ],+ },+ })++ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(+ () => undefined,+ (e: unknown) => e+ )++ // Witness: both workloads failed and were reported.+ expect((error as Error).message).toContain(+ 'Pre-deploy failed for workload(s): redis-mcp, files-mcp')+ expect(error).not.toBeInstanceOf(RetryableReconcileError)})it('returns empty array when recipe has no transport workloads', async () => {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex ee529d33b..f9cd6e780 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -24,7 +24,11 @@ import { createLogger } from '../observability/logger'import type { Logger } from '../observability/logger'import { WorkflowRecipeCRD, WorkloadDef } from '../types'import { CRD_GROUP, CRD_VERSION } from './crdConstants'-import { ResourceVanishedAfterConflictError, getErrorCode } from './k8sErrors'+import {+ ResourceVanishedAfterConflictError,+ RetryableReconcileError,+ getErrorCode,+} from './k8sErrors'import {ownerRef,resolveWorkloadResourceName,@@ -1098,7 +1102,7 @@ export async function ensureRecipeContext(}let wrote: boolean- if (initial) {+ if (initial !== null) {;({ wrote } = await replaceExistingContext(initial))} else {try {@@ -1230,9 +1234,13 @@ export async function preDeployMcpServers(}if (errors.length > 0) {const ids = errors.map(e => e.workloadId).join(', ')- throw new Error(- `Pre-deploy failed for workload(s): ${ids}. Reconciler will retry to ensure all NetworkPolicies are applied before workloads start.`- )+ const message = `Pre-deploy failed for workload(s): ${ids}. Reconciler will retry to ensure all NetworkPolicies are applied before workloads start.`+ // Retryable only when every failure was (e.g. a Service that vanished after+ // a create conflict); one permanent failure keeps the error terminal.+ if (errors.every(e => e.error instanceof RetryableReconcileError)) {+ throw new RetryableReconcileError(message, { cause: errors[0].error })+ }+ throw new Error(message)}// Create (or update) the per-recipe Context CRD immediately after McpServers@@ -1260,11 +1268,15 @@ export async function preDeployMcpServers(recipe.metadata.namespace)} catch (err) {- throw new Error(- `Pre-deploy Context allowlist failed for "${recipe.metadata.name}": ${- err instanceof Error ? err.message : String(err)- }`- )+ const message = `Pre-deploy Context allowlist failed for "${recipe.metadata.name}": ${+ err instanceof Error ? err.message : String(err)+ }`+ // Keep a retryable cause retryable, so the reconcile degrades and+ // requeues instead of latching `failed` (#998 audit).+ if (err instanceof RetryableReconcileError) {+ throw new RetryableReconcileError(message, { cause: err })+ }+ throw new Error(message)}}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex b1c04cdaf..f5f69b14c 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -4213,6 +4213,8 @@ describe('WorkflowRecipeReconciler', () => {const outcome = ensure()await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)+ // The cause is the 404 of the re-read, as in ensureTransportService.+ await expect(outcome).rejects.toMatchObject({ cause: { code: 404 } })const name = createdBody().metadata.nameexpect(readsOf(read, name, namespace)).toBe(2)expect(patch).toHaveBeenCalledTimes(0)diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 560e34f0b..ed044413e 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -4830,11 +4830,15 @@ export class WorkflowRecipeReconciler {})throw error}- // Another writer created it between our read and this POST.- existing = await this.readStatefulSetIfPresent(name, namespace)- if (!existing) {+ // Another writer created it between our read and this POST. A 404 on+ // this re-read means it is gone again; the 404 is the cause, as in+ // ensureTransportService.+ try {+ existing = await this.appsApi.readNamespacedStatefulSet({ name, namespace })+ } catch (readError: unknown) {+ if (getErrorCode(readError) !== 404) throw readErrorthrow new ResourceVanishedAfterConflictError(`StatefulSet "${name}" in ${namespace}`, {- cause: error,+ cause: readError,})}}