Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): retry the mcp-host Service replace once on a conflict

Alfredo Lopezc5511264bc9f
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index f7f4dd9e8..9ccb0173f 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 51d284208..7c16112ed 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index 10d022505..56f8eda77 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -3814,7 +3814,8 @@ export class WorkflowReconciler {
*
* Reads first and writes only when needed: creates the Service when it is
* absent, and replaces it when its selector or ports have drifted. A live
- * Service that matches is left untouched.
+ * Service that matches is left untouched. A replace that conflicts with a
+ * concurrent write is re-judged against a fresh read and retried once.
*/
private async ensureMcpHostHeadlessService(recipeName: string): Promise<void> {
const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)
@@ -3851,30 +3852,48 @@ export class WorkflowReconciler {
targetPort: p.targetPort ?? null,
protocol: p.protocol ?? 'TCP',
}))
- const desiredSpec = {
+ const desiredSpec = JSON.stringify({
selector: headlessSvc.spec?.selector ?? null,
ports: normalizePorts(headlessSvc.spec?.ports),
+ })
+ // A 409 on the replace means another writer changed the Service after the
+ // read, so its resourceVersion is stale. Re-read, judge the new live
+ // Service, and retry once, as applyNetworkPolicy does.
+ for (let attempt = 0; attempt < 2; attempt += 1) {
+ const existingSpec = JSON.stringify({
+ selector: existing.spec?.selector ?? null,
+ ports: normalizePorts(existing.spec?.ports),
+ })
+ if (desiredSpec === existingSpec) {
+ return
+ }
+ const updatedSvc: k8s.V1Service = {
+ ...headlessSvc,
+ metadata: {
+ ...headlessSvc.metadata,
+ resourceVersion: existing.metadata?.resourceVersion,
+ },
+ spec: {
+ ...headlessSvc.spec,
+ clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,
+ },
+ }
+ try {
+ await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
+ this.log.info(`Updated Headless Service "${name}"`)
+ return
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 409 || attempt === 1) throw error
+ const reread = await this.readServiceIfExists(name, namespace)
+ if (!reread) {
+ throw new RetryableReconcileError(
+ `Headless Service "${name}" disappeared after a replace conflict; a fresh reconciliation is required`,
+ { cause: error }
+ )
+ }
+ existing = reread
+ }
}
- const existingSpec = {
- selector: existing.spec?.selector ?? null,
- ports: normalizePorts(existing.spec?.ports),
- }
- if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {
- return
- }
- const updatedSvc: k8s.V1Service = {
- ...headlessSvc,
- metadata: {
- ...headlessSvc.metadata,
- resourceVersion: existing.metadata?.resourceVersion,
- },
- spec: {
- ...headlessSvc.spec,
- clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,
- },
- }
- await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
- this.log.info(`Updated Headless Service "${name}"`)
}
/**
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 6ec88e0cf..c6a16fe5e 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -3037,6 +3037,115 @@ describe('WorkflowReconciler — reconcile loop', () => {
expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
})
+ describe('when the drift replace conflicts with a concurrent write', () => {
+ const driftedMcpHost = (resourceVersion: string) => {
+ const desired = buildMcpHostHeadlessService('test-wf', namespace)
+ return asLiveService(
+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
+ resourceVersion
+ )
+ }
+
+ /** Answer successive mcp-host reads in order; a read past the list fails loudly. */
+ const serveMcpHostReads = (...responses: Array<LiveService | 404>) => {
+ let reads = 0
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
+ if (name !== mcpHostName) throw { code: 404 }
+ const response = responses[reads]
+ reads += 1
+ if (response === undefined) throw new Error(`unexpected read #${reads} of ${mcpHostName}`)
+ if (response === 404) throw { code: 404 }
+ return structuredClone(response)
+ })
+ }
+
+ const conflictOnFirstReplaces = (conflicts: number) => {
+ let replaces = 0
+ vi.mocked(deps.coreApi.replaceNamespacedService).mockImplementation(async ({ body }) => {
+ replaces += 1
+ if (replaces <= conflicts) throw { code: 409 }
+ return body
+ })
+ }
+
+ const replaceCalls = () => vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
+
+ it('re-reads and retries the replace once with the fresh resourceVersion', async () => {
+ serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
+ conflictOnFirstReplaces(1)
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([
+ 'rv-7',
+ 'rv-8',
+ ])
+ expect(replaceCalls()[1][0].body.spec?.ports?.map(port => port.name)).toEqual([
+ 'http',
+ 'plugin-sdk',
+ ])
+ })
+
+ it('stops without a second write when the re-read Service already matches', async () => {
+ serveMcpHostReads(
+ driftedMcpHost('rv-7'),
+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-8')
+ )
+ conflictOnFirstReplaces(1)
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(replaceCalls()).toHaveLength(1)
+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
+ })
+
+ it('asks for a fresh pass, without failing, when the Service vanishes after the conflict', async () => {
+ serveMcpHostReads(driftedMcpHost('rv-7'), 404)
+ conflictOnFirstReplaces(1)
+
+ const result = await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(replaceCalls()).toHaveLength(1)
+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(result.phase).not.toBe('failed')
+ expect(result.skipStatusPatch).toBe(true)
+ expect(result.message).toContain(
+ `Headless Service "${mcpHostName}" disappeared after a replace conflict`
+ )
+ })
+
+ it('fails the pass when the replace conflicts on both attempts', async () => {
+ serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
+ conflictOnFirstReplaces(2)
+
+ const result = await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(replaceCalls()).toHaveLength(2)
+ expect(result.workflowPhase).toBe('failed')
+ })
+
+ it('propagates a replace error other than a conflict without retrying', async () => {
+ serveMcpHostReads(driftedMcpHost('rv-7'))
+ vi.mocked(deps.coreApi.replaceNamespacedService).mockRejectedValue(
+ Object.assign(new Error('HTTP-Code: 422 Message: Service is invalid'), { code: 422 })
+ )
+
+ const result = await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
+ expect(replaceCalls()).toHaveLength(1)
+ expect(result.workflowPhase).toBe('failed')
+ expect(result.message).toContain('HTTP-Code: 422')
+ })
+ })
+
it('leaves a drifted route alias Service untouched because it is create-only', async () => {
const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)
serveLiveServices(deps, [