Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): retry when the mcp-host Service vanishes after a create conflict

Alfredo Lopez54e1238c7169
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 68eac6fd9..f7f4dd9e8 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.369",
+ "version": "0.1.370",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.369",
+ "version": "0.1.370",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index cbb672a79..51d284208 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.369",
+ "version": "0.1.370",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index 3df82a3c1..59e3a5f3e 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -1,4 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { buildMcpHostHeadlessService } from './podFactory'
+import { buildMcpHostServiceName } from './resourceNames'
import { WorkflowReconciler, type WorkflowReconcilerDeps } from './workflowReconciler'
const crashRecoveryMocks = vi.hoisted(() => ({
@@ -1587,6 +1589,41 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {
)
})
+ it('reads the eager mcp-host Service and sends no POST when the live one matches', async () => {
+ const serviceName = buildMcpHostServiceName('sdk-only')
+ const live = {
+ ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace),
+ metadata: {
+ ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace).metadata,
+ resourceVersion: 'rv-3',
+ },
+ }
+ mockCoreApi.readNamespacedService.mockImplementation(async ({ name }: { name: string }) => {
+ if (name === serviceName) return structuredClone(live)
+ throw { code: 404 }
+ })
+ const reconciler = new WorkflowReconciler(makeDeps())
+
+ const result = await reconciler.reconcilePluginWorkloadSdkOnly(
+ 'sdk-only',
+ 'uid-sdk-only',
+ sandboxNamespace,
+ sdkSpec({ steps: undefined })
+ )
+
+ // This fixture has no replaceNamespacedService, so a replace would throw
+ // and the phase would not be 'active'.
+ expect(result.phase).toBe('active')
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
+ name: serviceName,
+ namespace: sandboxNamespace,
+ })
+ const createdServiceNames = mockCoreApi.createNamespacedService.mock.calls.map(
+ call => call[0].body.metadata.name
+ )
+ expect(createdServiceNames).not.toContain(serviceName)
+ })
+
it('creates a provider-free eager mcp-host for clientNotifications-only without an agent', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index f123ebc26..10d022505 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -25,7 +25,12 @@ import {
import { mintRecipeHostGfsToken } from '../gfsBinding'
import { createLogger } from '../observability/logger'
import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from '../reconciler/crdConstants'
-import { getErrorCode, isRetryableInfraError } from '../reconciler/k8sErrors'
+import {
+ ResourceVanishedAfterConflictError,
+ RetryableReconcileError,
+ getErrorCode,
+ isRetryableInfraError,
+} from '../reconciler/k8sErrors'
import {
resolveStatefulSetHeadlessServiceName,
resolveWorkloadMcpServerLabel,
@@ -2424,8 +2429,10 @@ export class WorkflowReconciler {
// the terminal `failed` phase — that would brick a recoverable run with no
// retry. Preserve the current workflow execution phase/message and signal
// skipStatusPatch so WRC leaves status untouched and requeues. Mirrors the
- // outer WRC catch-all (isRetryableInfraError); same classifier.
- if (isRetryableInfraError(error)) {
+ // outer WRC catch-all (isRetryableInfraError); same classifier. A step that
+ // throws RetryableReconcileError (e.g. a Service that vanished after a
+ // create conflict) has declared itself transient and is treated the same.
+ if (error instanceof RetryableReconcileError || isRetryableInfraError(error)) {
log.warn(`Transient infra error reconciling workflow — will retry, not failing`, {
error: error instanceof Error ? error.message : String(error),
})
@@ -3801,9 +3808,13 @@ export class WorkflowReconciler {
}
/**
- * Create the mcp-host headless Service that backs its in-cluster DNS name
+ * Converge the mcp-host headless Service that backs its in-cluster DNS name
* (wf-<recipe>-mcp-host). Shared by the triggered-run and eager SDK paths;
* without it the SDK endpoint and coordinator→mcp-host calls fail at DNS.
+ *
+ * Reads first and writes only when needed: creates the Service when it is
+ * absent, and replaces it when its selector or ports have drifted. A live
+ * Service that matches is left untouched.
*/
private async ensureMcpHostHeadlessService(recipeName: string): Promise<void> {
const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)
@@ -3820,7 +3831,14 @@ export class WorkflowReconciler {
} catch (error: unknown) {
if (getErrorCode(error) !== 409) throw error
// Another writer created it between the read and the POST.
- existing = await this.deps.coreApi.readNamespacedService({ name, namespace })
+ existing = await this.readServiceIfExists(name, namespace)
+ if (!existing) {
+ // ...and it was deleted again before the re-read. There is no live
+ // Service to judge, so this pass stops and asks for a fresh one.
+ throw new ResourceVanishedAfterConflictError(`Headless Service "${name}"`, {
+ cause: error,
+ })
+ }
}
}
// Skip the replace when the live Service already matches the desired
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index fcfc16711..6ec88e0cf 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -2992,19 +2992,30 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
- it('reads, then creates the mcp-host Service exactly once when it is absent', async () => {
+ it('reads, then creates the mcp-host and route alias Services exactly once when they are absent', async () => {
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
+ expect(serviceReadsOf(deps, aliasName)).toBe(1)
expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
+ expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {
const desired = buildMcpHostHeadlessService('test-wf', namespace)
+ // clusterIP differs from the desired 'None' so the assertion below can
+ // only pass if the replace carries the live (immutable) value over.
const drifted = asLiveService(
- { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
+ {
+ ...desired,
+ spec: {
+ ...desired.spec,
+ clusterIP: '10.96.0.42',
+ ports: desired.spec?.ports?.slice(0, 1),
+ },
+ },
'rv-7'
)
serveLiveServices(deps, [
@@ -3022,10 +3033,31 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(name).toBe(mcpHostName)
expect(replacedIn).toBe(namespace)
expect(body.metadata?.resourceVersion).toBe('rv-7')
- expect(body.spec?.clusterIP).toBe(drifted.spec?.clusterIP)
+ expect(body.spec?.clusterIP).toBe('10.96.0.42')
expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
})
+ it('leaves a drifted route alias Service untouched because it is create-only', async () => {
+ const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)
+ serveLiveServices(deps, [
+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
+ asLiveService(
+ {
+ ...desiredAlias,
+ spec: { ...desiredAlias.spec, ports: desiredAlias.spec?.ports?.slice(0, 1) },
+ },
+ 'rv-2'
+ ),
+ ])
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, aliasName)).toBe(1)
+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {
const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')
let mcpHostReads = 0
@@ -3048,8 +3080,60 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
+ it('repairs a drifted mcp-host Service that another writer created after the read', async () => {
+ const desired = buildMcpHostHeadlessService('test-wf', namespace)
+ const racedDrifted = asLiveService(
+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
+ 'rv-11'
+ )
+ let mcpHostReads = 0
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
+ if (name !== mcpHostName) throw { code: 404 }
+ mcpHostReads += 1
+ if (mcpHostReads === 1) throw { code: 404 }
+ return structuredClone(racedDrifted)
+ })
+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
+ if (body.metadata?.name === mcpHostName) throw { code: 409 }
+ return body
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
+ expect(replaceCalls).toHaveLength(1)
+ const [{ name, body }] = replaceCalls[0]
+ expect(name).toBe(mcpHostName)
+ expect(body.metadata?.resourceVersion).toBe('rv-11')
+ expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
+ })
+
+ it('asks for a fresh pass, without failing, when the mcp-host Service vanishes after a create conflict', async () => {
+ vi.mocked(deps.coreApi.readNamespacedService).mockRejectedValue({ code: 404 })
+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
+ if (body.metadata?.name === mcpHostName) throw { code: 409 }
+ return body
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(result.phase).not.toBe('failed')
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.message).toContain(
+ `Headless Service "${mcpHostName}" disappeared after create conflict`
+ )
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {
- const forbidden = { code: 403 }
+ const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
+ code: 403,
+ })
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name === mcpHostName) throw forbidden
throw { code: 404 }
@@ -3059,9 +3143,43 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
expect(result.workflowPhase).toBe('failed')
+ expect(result.message).toContain('HTTP-Code: 403')
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
})
+ it('propagates a non-404 read error instead of creating a create-only Service blind', async () => {
+ const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
+ code: 403,
+ })
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
+ if (name === aliasName) throw forbidden
+ throw { code: 404 }
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ // The mcp-host Service is handled first and was created, so the pass reached the alias.
+ expect(createdServiceNamesOf(deps)).toContain(mcpHostName)
+ expect(serviceReadsOf(deps, aliasName)).toBe(1)
+ expect(result.workflowPhase).toBe('failed')
+ expect(result.message).toContain('HTTP-Code: 403')
+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
+ })
+
+ it('converges when another writer creates a create-only Service between the read and the POST', async () => {
+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
+ if (body.metadata?.name === aliasName) throw { code: 409 }
+ return body
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, aliasName)).toBe(1)
+ expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
describe('snippet-runner and artifact-reader Services', () => {
const reconcileSnippetWorkflow = () => {
vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({