Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): retry the mcp-host Service replace once on a conflict
Alfredo Lopezc5511264bc9f
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex f7f4dd9e8..9ccb0173f 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.370",+ "version": "0.1.371","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.370",+ "version": "0.1.371","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 51d284208..7c16112ed 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.370",+ "version": "0.1.371","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex 10d022505..56f8eda77 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -3814,7 +3814,8 @@ export class WorkflowReconciler {** Reads first and writes only when needed: creates the Service when it is* absent, and replaces it when its selector or ports have drifted. A live- * Service that matches is left untouched.+ * Service that matches is left untouched. A replace that conflicts with a+ * concurrent write is re-judged against a fresh read and retried once.*/private async ensureMcpHostHeadlessService(recipeName: string): Promise<void> {const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)@@ -3851,30 +3852,48 @@ export class WorkflowReconciler {targetPort: p.targetPort ?? null,protocol: p.protocol ?? 'TCP',}))- const desiredSpec = {+ const desiredSpec = JSON.stringify({selector: headlessSvc.spec?.selector ?? null,ports: normalizePorts(headlessSvc.spec?.ports),+ })+ // A 409 on the replace means another writer changed the Service after the+ // read, so its resourceVersion is stale. Re-read, judge the new live+ // Service, and retry once, as applyNetworkPolicy does.+ for (let attempt = 0; attempt < 2; attempt += 1) {+ const existingSpec = JSON.stringify({+ selector: existing.spec?.selector ?? null,+ ports: normalizePorts(existing.spec?.ports),+ })+ if (desiredSpec === existingSpec) {+ return+ }+ const updatedSvc: k8s.V1Service = {+ ...headlessSvc,+ metadata: {+ ...headlessSvc.metadata,+ resourceVersion: existing.metadata?.resourceVersion,+ },+ spec: {+ ...headlessSvc.spec,+ clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,+ },+ }+ try {+ await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })+ this.log.info(`Updated Headless Service "${name}"`)+ return+ } catch (error: unknown) {+ if (getErrorCode(error) !== 409 || attempt === 1) throw error+ const reread = await this.readServiceIfExists(name, namespace)+ if (!reread) {+ throw new RetryableReconcileError(+ `Headless Service "${name}" disappeared after a replace conflict; a fresh reconciliation is required`,+ { cause: error }+ )+ }+ existing = reread+ }}- const existingSpec = {- selector: existing.spec?.selector ?? null,- ports: normalizePorts(existing.spec?.ports),- }- if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {- return- }- const updatedSvc: k8s.V1Service = {- ...headlessSvc,- metadata: {- ...headlessSvc.metadata,- resourceVersion: existing.metadata?.resourceVersion,- },- spec: {- ...headlessSvc.spec,- clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,- },- }- await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })- this.log.info(`Updated Headless Service "${name}"`)}/**diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 6ec88e0cf..c6a16fe5e 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -3037,6 +3037,115 @@ describe('WorkflowReconciler — reconcile loop', () => {expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])})+ describe('when the drift replace conflicts with a concurrent write', () => {+ const driftedMcpHost = (resourceVersion: string) => {+ const desired = buildMcpHostHeadlessService('test-wf', namespace)+ return asLiveService(+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },+ resourceVersion+ )+ }++ /** Answer successive mcp-host reads in order; a read past the list fails loudly. */+ const serveMcpHostReads = (...responses: Array<LiveService | 404>) => {+ let reads = 0+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {+ if (name !== mcpHostName) throw { code: 404 }+ const response = responses[reads]+ reads += 1+ if (response === undefined) throw new Error(`unexpected read #${reads} of ${mcpHostName}`)+ if (response === 404) throw { code: 404 }+ return structuredClone(response)+ })+ }++ const conflictOnFirstReplaces = (conflicts: number) => {+ let replaces = 0+ vi.mocked(deps.coreApi.replaceNamespacedService).mockImplementation(async ({ body }) => {+ replaces += 1+ if (replaces <= conflicts) throw { code: 409 }+ return body+ })+ }++ const replaceCalls = () => vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls++ it('re-reads and retries the replace once with the fresh resourceVersion', async () => {+ serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))+ conflictOnFirstReplaces(1)++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([+ 'rv-7',+ 'rv-8',+ ])+ expect(replaceCalls()[1][0].body.spec?.ports?.map(port => port.name)).toEqual([+ 'http',+ 'plugin-sdk',+ ])+ })++ it('stops without a second write when the re-read Service already matches', async () => {+ serveMcpHostReads(+ driftedMcpHost('rv-7'),+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-8')+ )+ conflictOnFirstReplaces(1)++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(replaceCalls()).toHaveLength(1)+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)+ })++ it('asks for a fresh pass, without failing, when the Service vanishes after the conflict', async () => {+ serveMcpHostReads(driftedMcpHost('rv-7'), 404)+ conflictOnFirstReplaces(1)++ const result = await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(replaceCalls()).toHaveLength(1)+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)+ expect(result.workflowPhase).not.toBe('failed')+ expect(result.phase).not.toBe('failed')+ expect(result.skipStatusPatch).toBe(true)+ expect(result.message).toContain(+ `Headless Service "${mcpHostName}" disappeared after a replace conflict`+ )+ })++ it('fails the pass when the replace conflicts on both attempts', async () => {+ serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))+ conflictOnFirstReplaces(2)++ const result = await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(replaceCalls()).toHaveLength(2)+ expect(result.workflowPhase).toBe('failed')+ })++ it('propagates a replace error other than a conflict without retrying', async () => {+ serveMcpHostReads(driftedMcpHost('rv-7'))+ vi.mocked(deps.coreApi.replaceNamespacedService).mockRejectedValue(+ Object.assign(new Error('HTTP-Code: 422 Message: Service is invalid'), { code: 422 })+ )++ const result = await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)+ expect(replaceCalls()).toHaveLength(1)+ expect(result.workflowPhase).toBe('failed')+ expect(result.message).toContain('HTTP-Code: 422')+ })+ })+it('leaves a drifted route alias Service untouched because it is create-only', async () => {const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)serveLiveServices(deps, [