Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): keep pre-deploy vanish errors retryable (#760)

Alfredo Lopezfcc914a82b21
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index f91d0bc0b..8b72a97bb 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.466",
+ "version": "0.1.467",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.466",
+ "version": "0.1.467",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 2aa529698..6b07cc1bb 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.466",
+ "version": "0.1.467",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
index 1360e483b..21a1d6453 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
@@ -6,7 +6,7 @@
import { describe, expect, it, vi } from 'vitest'
import { WorkflowRecipeCRD } from '../types'
import { captureLogger, captureLoggerLevels } from './__tests__/captureLogger'
-import { ResourceVanishedAfterConflictError } from './k8sErrors'
+import { ResourceVanishedAfterConflictError, RetryableReconcileError } from './k8sErrors'
import {
DelegationDeps,
PRE_DEPLOY_ANNOTATION,
@@ -1142,7 +1142,7 @@ describe('ensureRecipeContext', () => {
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalled()
})
- it('H04b — writes on 409 when live Context has no mcpServers (empty→filled is not a skip)', async () => {
+ it('H04b — replaces an existing Context that has no mcpServers (empty→filled is not a skip)', async () => {
const mockCustomApi = {
createNamespacedCustomObject: vi.fn(),
getNamespacedCustomObject: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '5' } }),
@@ -3029,9 +3029,10 @@ describe('preDeployMcpServers', () => {
})
const recipe = makeRecipe()
- await expect(preDeployMcpServers(deps, recipe, 'mcp-server', new Map())).rejects.toThrow(
- 'Pre-deploy failed for workload(s): redis-mcp'
- )
+ const outcome = preDeployMcpServers(deps, recipe, 'mcp-server', new Map())
+ await expect(outcome).rejects.toThrow('Pre-deploy failed for workload(s): redis-mcp')
+ // A permanent failure stays terminal (#998 audit).
+ await expect(outcome).rejects.not.toBeInstanceOf(RetryableReconcileError)
})
it('throws when the pre-deploy Context allowlist cannot be persisted', async () => {
@@ -3053,9 +3054,123 @@ describe('preDeployMcpServers', () => {
})
const recipe = makeRecipe()
- await expect(preDeployMcpServers(deps, recipe, 'mcp-server', new Map())).rejects.toThrow(
- 'Pre-deploy Context allowlist failed'
+ const outcome = preDeployMcpServers(deps, recipe, 'mcp-server', new Map())
+ await expect(outcome).rejects.toThrow('Pre-deploy Context allowlist failed')
+ // A permanent Context error stays terminal (#998 audit).
+ await expect(outcome).rejects.not.toBeInstanceOf(RetryableReconcileError)
+ })
+
+ it('keeps a Context that vanished after a create conflict retryable on the pre-deploy path', async () => {
+ // GET 404, POST 409, re-read 404: the writer raises ResourceVanishedAfterConflictError.
+ // The pre-deploy wrapper must not turn it into a terminal error (#998 audit).
+ const deps = makePreDeployDeps({
+ customApi: {
+ createNamespacedCustomObject: vi
+ .fn()
+ .mockImplementation((opts: { body: { kind?: string } }) =>
+ opts.body.kind === 'Context' ? Promise.reject({ code: 409 }) : Promise.resolve({})
+ ),
+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),
+ patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+ deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
+ } as any,
+ })
+ const recipe = makeRecipe()
+
+ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(
+ () => undefined,
+ (e: unknown) => e
+ )
+
+ expect(error).toBeInstanceOf(RetryableReconcileError)
+ expect((error as Error).message).toContain('Pre-deploy Context allowlist failed')
+ expect((error as Error).message).toContain('disappeared after create conflict')
+ expect((error as { cause?: unknown }).cause).toBeInstanceOf(ResourceVanishedAfterConflictError)
+ })
+
+ it('keeps a pre-deploy failure retryable when every workload failed retryably', async () => {
+ // Service GET 404, POST 409, re-read 404 for the only transport workload.
+ const deps = makePreDeployDeps({
+ coreApi: {
+ createNamespacedService: vi.fn().mockRejectedValue({ code: 409 }),
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ replaceNamespacedService: vi.fn().mockResolvedValue({}),
+ deleteNamespacedService: vi.fn().mockResolvedValue({}),
+ } as any,
+ })
+ const recipe = makeRecipe()
+
+ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(
+ () => undefined,
+ (e: unknown) => e
+ )
+
+ expect(error).toBeInstanceOf(RetryableReconcileError)
+ expect((error as Error).message).toContain('Pre-deploy failed for workload(s): redis-mcp')
+ })
+
+ it('keeps a pre-deploy failure terminal when any workload failed permanently', async () => {
+ // redis-mcp: its Service vanishes after a create conflict (retryable).
+ // files-mcp: its McpServer create fails with a permanent error.
+ const deps = makePreDeployDeps({
+ customApi: {
+ createNamespacedCustomObject: vi
+ .fn()
+ .mockImplementation((opts: { body: { kind?: string; metadata?: { name?: string } } }) =>
+ opts.body.kind === 'McpServer' && String(opts.body.metadata?.name).includes('files-mcp')
+ ? Promise.reject(new Error('quota exceeded'))
+ : Promise.resolve({})
+ ),
+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),
+ patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+ deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+ listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
+ } as any,
+ coreApi: {
+ createNamespacedService: vi
+ .fn()
+ .mockImplementation((opts: { body: { metadata?: { name?: string } } }) =>
+ String(opts.body.metadata?.name).includes('redis-mcp')
+ ? Promise.reject({ code: 409 })
+ : Promise.resolve({})
+ ),
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ replaceNamespacedService: vi.fn().mockResolvedValue({}),
+ deleteNamespacedService: vi.fn().mockResolvedValue({}),
+ } as any,
+ })
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'redis-mcp',
+ type: 'deployment',
+ image: 'clerum/redis-mcp:latest',
+ port: 3000,
+ transport: { type: 'streamableHttp', path: '/mcp' },
+ },
+ {
+ id: 'files-mcp',
+ type: 'deployment',
+ image: 'clerum/files-mcp:latest',
+ port: 3000,
+ transport: { type: 'streamableHttp', path: '/mcp' },
+ },
+ ],
+ },
+ })
+
+ const error = await preDeployMcpServers(deps, recipe, 'mcp-server', new Map()).then(
+ () => undefined,
+ (e: unknown) => e
+ )
+
+ // Witness: both workloads failed and were reported.
+ expect((error as Error).message).toContain(
+ 'Pre-deploy failed for workload(s): redis-mcp, files-mcp'
)
+ expect(error).not.toBeInstanceOf(RetryableReconcileError)
})
it('returns empty array when recipe has no transport workloads', async () => {
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.ts
index ee529d33b..f9cd6e780 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts
@@ -24,7 +24,11 @@ import { createLogger } from '../observability/logger'
import type { Logger } from '../observability/logger'
import { WorkflowRecipeCRD, WorkloadDef } from '../types'
import { CRD_GROUP, CRD_VERSION } from './crdConstants'
-import { ResourceVanishedAfterConflictError, getErrorCode } from './k8sErrors'
+import {
+ ResourceVanishedAfterConflictError,
+ RetryableReconcileError,
+ getErrorCode,
+} from './k8sErrors'
import {
ownerRef,
resolveWorkloadResourceName,
@@ -1098,7 +1102,7 @@ export async function ensureRecipeContext(
}
let wrote: boolean
- if (initial) {
+ if (initial !== null) {
;({ wrote } = await replaceExistingContext(initial))
} else {
try {
@@ -1230,9 +1234,13 @@ export async function preDeployMcpServers(
}
if (errors.length > 0) {
const ids = errors.map(e => e.workloadId).join(', ')
- throw new Error(
- `Pre-deploy failed for workload(s): ${ids}. Reconciler will retry to ensure all NetworkPolicies are applied before workloads start.`
- )
+ const message = `Pre-deploy failed for workload(s): ${ids}. Reconciler will retry to ensure all NetworkPolicies are applied before workloads start.`
+ // Retryable only when every failure was (e.g. a Service that vanished after
+ // a create conflict); one permanent failure keeps the error terminal.
+ if (errors.every(e => e.error instanceof RetryableReconcileError)) {
+ throw new RetryableReconcileError(message, { cause: errors[0].error })
+ }
+ throw new Error(message)
}
// Create (or update) the per-recipe Context CRD immediately after McpServers
@@ -1260,11 +1268,15 @@ export async function preDeployMcpServers(
recipe.metadata.namespace
)
} catch (err) {
- throw new Error(
- `Pre-deploy Context allowlist failed for "${recipe.metadata.name}": ${
- err instanceof Error ? err.message : String(err)
- }`
- )
+ const message = `Pre-deploy Context allowlist failed for "${recipe.metadata.name}": ${
+ err instanceof Error ? err.message : String(err)
+ }`
+ // Keep a retryable cause retryable, so the reconcile degrades and
+ // requeues instead of latching `failed` (#998 audit).
+ if (err instanceof RetryableReconcileError) {
+ throw new RetryableReconcileError(message, { cause: err })
+ }
+ throw new Error(message)
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index b1c04cdaf..f5f69b14c 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -4213,6 +4213,8 @@ describe('WorkflowRecipeReconciler', () => {
const outcome = ensure()
await expect(outcome).rejects.toBeInstanceOf(ResourceVanishedAfterConflictError)
+ // The cause is the 404 of the re-read, as in ensureTransportService.
+ await expect(outcome).rejects.toMatchObject({ cause: { code: 404 } })
const name = createdBody().metadata.name
expect(readsOf(read, name, namespace)).toBe(2)
expect(patch).toHaveBeenCalledTimes(0)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 560e34f0b..ed044413e 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -4830,11 +4830,15 @@ export class WorkflowRecipeReconciler {
})
throw error
}
- // Another writer created it between our read and this POST.
- existing = await this.readStatefulSetIfPresent(name, namespace)
- if (!existing) {
+ // Another writer created it between our read and this POST. A 404 on
+ // this re-read means it is gone again; the 404 is the cause, as in
+ // ensureTransportService.
+ try {
+ existing = await this.appsApi.readNamespacedStatefulSet({ name, namespace })
+ } catch (readError: unknown) {
+ if (getErrorCode(readError) !== 404) throw readError
throw new ResourceVanishedAfterConflictError(`StatefulSet "${name}" in ${namespace}`, {
- cause: error,
+ cause: readError,
})
}
}