Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): retry when the mcp-host Service vanishes after a create conflict
Alfredo Lopez54e1238c7169
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 68eac6fd9..f7f4dd9e8 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.369",+ "version": "0.1.370","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.369",+ "version": "0.1.370","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex cbb672a79..51d284208 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.369",+ "version": "0.1.370","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex 3df82a3c1..59e3a5f3e 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -1,4 +1,6 @@import { beforeEach, describe, expect, it, vi } from 'vitest'+import { buildMcpHostHeadlessService } from './podFactory'+import { buildMcpHostServiceName } from './resourceNames'import { WorkflowReconciler, type WorkflowReconcilerDeps } from './workflowReconciler'const crashRecoveryMocks = vi.hoisted(() => ({@@ -1587,6 +1589,41 @@ describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {)})+ it('reads the eager mcp-host Service and sends no POST when the live one matches', async () => {+ const serviceName = buildMcpHostServiceName('sdk-only')+ const live = {+ ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace),+ metadata: {+ ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace).metadata,+ resourceVersion: 'rv-3',+ },+ }+ mockCoreApi.readNamespacedService.mockImplementation(async ({ name }: { name: string }) => {+ if (name === serviceName) return structuredClone(live)+ throw { code: 404 }+ })+ const reconciler = new WorkflowReconciler(makeDeps())++ const result = await reconciler.reconcilePluginWorkloadSdkOnly(+ 'sdk-only',+ 'uid-sdk-only',+ sandboxNamespace,+ sdkSpec({ steps: undefined })+ )++ // This fixture has no replaceNamespacedService, so a replace would throw+ // and the phase would not be 'active'.+ expect(result.phase).toBe('active')+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({+ name: serviceName,+ namespace: sandboxNamespace,+ })+ const createdServiceNames = mockCoreApi.createNamespacedService.mock.calls.map(+ call => call[0].body.metadata.name+ )+ expect(createdServiceNames).not.toContain(serviceName)+ })+it('creates a provider-free eager mcp-host for clientNotifications-only without an agent', async () => {const reconciler = new WorkflowReconciler(makeDeps())mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex f123ebc26..10d022505 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -25,7 +25,12 @@ import {import { mintRecipeHostGfsToken } from '../gfsBinding'import { createLogger } from '../observability/logger'import { CRD_GROUP, CRD_VERSION, WORKFLOWRECIPE_PLURAL } from '../reconciler/crdConstants'-import { getErrorCode, isRetryableInfraError } from '../reconciler/k8sErrors'+import {+ ResourceVanishedAfterConflictError,+ RetryableReconcileError,+ getErrorCode,+ isRetryableInfraError,+} from '../reconciler/k8sErrors'import {resolveStatefulSetHeadlessServiceName,resolveWorkloadMcpServerLabel,@@ -2424,8 +2429,10 @@ export class WorkflowReconciler {// the terminal `failed` phase — that would brick a recoverable run with no// retry. Preserve the current workflow execution phase/message and signal// skipStatusPatch so WRC leaves status untouched and requeues. Mirrors the- // outer WRC catch-all (isRetryableInfraError); same classifier.- if (isRetryableInfraError(error)) {+ // outer WRC catch-all (isRetryableInfraError); same classifier. A step that+ // throws RetryableReconcileError (e.g. a Service that vanished after a+ // create conflict) has declared itself transient and is treated the same.+ if (error instanceof RetryableReconcileError || isRetryableInfraError(error)) {log.warn(`Transient infra error reconciling workflow — will retry, not failing`, {error: error instanceof Error ? error.message : String(error),})@@ -3801,9 +3808,13 @@ export class WorkflowReconciler {}/**- * Create the mcp-host headless Service that backs its in-cluster DNS name+ * Converge the mcp-host headless Service that backs its in-cluster DNS name* (wf-<recipe>-mcp-host). Shared by the triggered-run and eager SDK paths;* without it the SDK endpoint and coordinator→mcp-host calls fail at DNS.+ *+ * Reads first and writes only when needed: creates the Service when it is+ * absent, and replaces it when its selector or ports have drifted. A live+ * Service that matches is left untouched.*/private async ensureMcpHostHeadlessService(recipeName: string): Promise<void> {const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)@@ -3820,7 +3831,14 @@ export class WorkflowReconciler {} catch (error: unknown) {if (getErrorCode(error) !== 409) throw error// Another writer created it between the read and the POST.- existing = await this.deps.coreApi.readNamespacedService({ name, namespace })+ existing = await this.readServiceIfExists(name, namespace)+ if (!existing) {+ // ...and it was deleted again before the re-read. There is no live+ // Service to judge, so this pass stops and asks for a fresh one.+ throw new ResourceVanishedAfterConflictError(`Headless Service "${name}"`, {+ cause: error,+ })+ }}}// Skip the replace when the live Service already matches the desireddiff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex fcfc16711..6ec88e0cf 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -2992,19 +2992,30 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)})- it('reads, then creates the mcp-host Service exactly once when it is absent', async () => {+ it('reads, then creates the mcp-host and route alias Services exactly once when they are absent', async () => {const result = await reconcileTriggeredRun()expect(result.workflowPhase).not.toBe('failed')expect(serviceReadsOf(deps, mcpHostName)).toBe(1)+ expect(serviceReadsOf(deps, aliasName)).toBe(1)expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)+ expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)})it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {const desired = buildMcpHostHeadlessService('test-wf', namespace)+ // clusterIP differs from the desired 'None' so the assertion below can+ // only pass if the replace carries the live (immutable) value over.const drifted = asLiveService(- { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },+ {+ ...desired,+ spec: {+ ...desired.spec,+ clusterIP: '10.96.0.42',+ ports: desired.spec?.ports?.slice(0, 1),+ },+ },'rv-7')serveLiveServices(deps, [@@ -3022,10 +3033,31 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(name).toBe(mcpHostName)expect(replacedIn).toBe(namespace)expect(body.metadata?.resourceVersion).toBe('rv-7')- expect(body.spec?.clusterIP).toBe(drifted.spec?.clusterIP)+ expect(body.spec?.clusterIP).toBe('10.96.0.42')expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])})+ it('leaves a drifted route alias Service untouched because it is create-only', async () => {+ const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)+ serveLiveServices(deps, [+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),+ asLiveService(+ {+ ...desiredAlias,+ spec: { ...desiredAlias.spec, ports: desiredAlias.spec?.ports?.slice(0, 1) },+ },+ 'rv-2'+ ),+ ])++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, aliasName)).toBe(1)+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })+it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')let mcpHostReads = 0@@ -3048,8 +3080,60 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)})+ it('repairs a drifted mcp-host Service that another writer created after the read', async () => {+ const desired = buildMcpHostHeadlessService('test-wf', namespace)+ const racedDrifted = asLiveService(+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },+ 'rv-11'+ )+ let mcpHostReads = 0+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {+ if (name !== mcpHostName) throw { code: 404 }+ mcpHostReads += 1+ if (mcpHostReads === 1) throw { code: 404 }+ return structuredClone(racedDrifted)+ })+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {+ if (body.metadata?.name === mcpHostName) throw { code: 409 }+ return body+ })++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls+ expect(replaceCalls).toHaveLength(1)+ const [{ name, body }] = replaceCalls[0]+ expect(name).toBe(mcpHostName)+ expect(body.metadata?.resourceVersion).toBe('rv-11')+ expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])+ })++ it('asks for a fresh pass, without failing, when the mcp-host Service vanishes after a create conflict', async () => {+ vi.mocked(deps.coreApi.readNamespacedService).mockRejectedValue({ code: 404 })+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {+ if (body.metadata?.name === mcpHostName) throw { code: 409 }+ return body+ })++ const result = await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)+ expect(result.workflowPhase).not.toBe('failed')+ expect(result.phase).not.toBe('failed')+ expect(result.skipStatusPatch).toBe(true)+ expect(result.message).toContain(+ `Headless Service "${mcpHostName}" disappeared after create conflict`+ )+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })+it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {- const forbidden = { code: 403 }+ const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {+ code: 403,+ })vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {if (name === mcpHostName) throw forbiddenthrow { code: 404 }@@ -3059,9 +3143,43 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(serviceReadsOf(deps, mcpHostName)).toBe(1)expect(result.workflowPhase).toBe('failed')+ expect(result.message).toContain('HTTP-Code: 403')expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)})+ it('propagates a non-404 read error instead of creating a create-only Service blind', async () => {+ const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {+ code: 403,+ })+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {+ if (name === aliasName) throw forbidden+ throw { code: 404 }+ })++ const result = await reconcileTriggeredRun()++ // The mcp-host Service is handled first and was created, so the pass reached the alias.+ expect(createdServiceNamesOf(deps)).toContain(mcpHostName)+ expect(serviceReadsOf(deps, aliasName)).toBe(1)+ expect(result.workflowPhase).toBe('failed')+ expect(result.message).toContain('HTTP-Code: 403')+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)+ })++ it('converges when another writer creates a create-only Service between the read and the POST', async () => {+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {+ if (body.metadata?.name === aliasName) throw { code: 409 }+ return body+ })++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, aliasName)).toBe(1)+ expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })+describe('snippet-runner and artifact-reader Services', () => {const reconcileSnippetWorkflow = () => {vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({