Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): read headless Services before writing them

Alfredo Lopez5c1e97b742ab
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 9dfd0c21a..68eac6fd9 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 774903918..cbb672a79 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.368",
+ "version": "0.1.369",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
index 04bad1016..02b9ab08f 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts
@@ -151,7 +151,7 @@ function createHarness() {
return {}
}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
- readNamespacedService: vi.fn().mockResolvedValue({}),
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedConfigMap: vi.fn().mockResolvedValue(eligibleCodexConfigMap()),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
index af4e19ab1..c59dc27c9 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts
@@ -90,7 +90,7 @@ function createHarness(grokSubscriptionEnabled: boolean, pluginWorkloadSdkEnable
return {}
}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
- readNamespacedService: vi.fn().mockResolvedValue({}),
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedConfigMap: vi.fn().mockResolvedValue(eligibleGrokConfigMap()),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.ts
index f41ca9c17..3df82a3c1 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts
@@ -116,7 +116,7 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
const mockCoreApi = {
readNamespacedPod: vi.fn().mockResolvedValue({}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
- readNamespacedService: vi.fn().mockResolvedValue({}),
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
@@ -200,7 +200,9 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
return {}
})
mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
- mockCoreApi.readNamespacedService.mockResolvedValue({})
+ // No Service exists until a test says so: reconcile reads before it
+ // creates, so a resolved read would stand for a live (spec-less) Service.
+ mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
mockCoreApi.createNamespacedSecret.mockResolvedValue({})
mockCoreApi.createNamespacedConfigMap.mockResolvedValue({})
mockCoreApi.createNamespacedService.mockResolvedValue({})
@@ -483,6 +485,7 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =
})
it('prefers artifact-reader HTTP artifact cleanup when the artifact-reader Service exists', async () => {
+ mockCoreApi.readNamespacedService.mockResolvedValue({})
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'agentic-recipe'
diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.ts
index b4df1587e..f123ebc26 100644
--- a/workflow-recipes/src/workflow/workflowReconciler.ts
+++ b/workflow-recipes/src/workflow/workflowReconciler.ts
@@ -2096,45 +2096,21 @@ export class WorkflowReconciler {
if (needsMcpHost && !awaitsTriggeredRun) {
await this.ensureMcpHostHeadlessService(recipeName)
- const routeAliasSvc = buildMcpHostRouteAliasHeadlessService(
- recipeName,
- this.deps.config.sandboxNamespace
- )
- await this.createIfNotExists(
- () =>
- this.deps.coreApi.createNamespacedService({
- namespace: this.deps.config.sandboxNamespace,
- body: routeAliasSvc,
- }),
- `Headless Service "${buildMcpHostRouteAliasServiceName(recipeName, this.deps.config.sandboxNamespace)}"`
+ await this.ensureHeadlessServiceExists(
+ buildMcpHostRouteAliasServiceName(recipeName, this.deps.config.sandboxNamespace),
+ buildMcpHostRouteAliasHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
if (needsArtifactReader && !awaitsTriggeredRun) {
- const readerSvc = buildArtifactReaderHeadlessService(
- recipeName,
- this.deps.config.sandboxNamespace
- )
- await this.createIfNotExists(
- () =>
- this.deps.coreApi.createNamespacedService({
- namespace: this.deps.config.sandboxNamespace,
- body: readerSvc,
- }),
- `Headless Service "${buildArtifactReaderServiceName(recipeName)}"`
+ await this.ensureHeadlessServiceExists(
+ buildArtifactReaderServiceName(recipeName),
+ buildArtifactReaderHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
if (needsSnippetRunner && !awaitsTriggeredRun) {
- const snippetRunnerSvc = buildSnippetRunnerHeadlessService(
- recipeName,
- this.deps.config.sandboxNamespace
- )
- await this.createIfNotExists(
- () =>
- this.deps.coreApi.createNamespacedService({
- namespace: this.deps.config.sandboxNamespace,
- body: snippetRunnerSvc,
- }),
- `Headless Service "${buildSnippetRunnerServiceName(recipeName)}"`
+ await this.ensureHeadlessServiceExists(
+ buildSnippetRunnerServiceName(recipeName),
+ buildSnippetRunnerHeadlessService(recipeName, this.deps.config.sandboxNamespace)
)
}
@@ -3833,46 +3809,78 @@ export class WorkflowReconciler {
const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)
const namespace = this.deps.config.sandboxNamespace
const name = buildMcpHostServiceName(recipeName)
- try {
- await this.deps.coreApi.createNamespacedService({ namespace, body: headlessSvc })
- this.log.info(`Created Headless Service "${name}"`)
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) throw error
- const existing = await this.deps.coreApi.readNamespacedService({ name, namespace })
- // Skip the replace when the live Service already matches the desired
- // spec — this method runs on every eager reconcile, and an unconditional
- // GET+PUT churns resourceVersions and apiserver writes for no drift.
- const normalizePorts = (ports: k8s.V1ServicePort[] | undefined) =>
- (ports ?? []).map(p => ({
- name: p.name ?? null,
- port: p.port,
- targetPort: p.targetPort ?? null,
- protocol: p.protocol ?? 'TCP',
- }))
- const desiredSpec = {
- selector: headlessSvc.spec?.selector ?? null,
- ports: normalizePorts(headlessSvc.spec?.ports),
- }
- const existingSpec = {
- selector: existing.spec?.selector ?? null,
- ports: normalizePorts(existing.spec?.ports),
- }
- if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {
+ // Read first: this method runs on every eager reconcile, and a POST used as
+ // an existence probe is an apiserver write (409) on every pass.
+ let existing = await this.readServiceIfExists(name, namespace)
+ if (!existing) {
+ try {
+ await this.deps.coreApi.createNamespacedService({ namespace, body: headlessSvc })
+ this.log.info(`Created Headless Service "${name}"`)
return
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 409) throw error
+ // Another writer created it between the read and the POST.
+ existing = await this.deps.coreApi.readNamespacedService({ name, namespace })
}
- const updatedSvc: k8s.V1Service = {
- ...headlessSvc,
- metadata: {
- ...headlessSvc.metadata,
- resourceVersion: existing.metadata?.resourceVersion,
- },
- spec: {
- ...headlessSvc.spec,
- clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,
- },
- }
- await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
- this.log.info(`Updated Headless Service "${name}"`)
+ }
+ // Skip the replace when the live Service already matches the desired
+ // spec — an unconditional PUT churns resourceVersions and apiserver writes
+ // for no drift.
+ const normalizePorts = (ports: k8s.V1ServicePort[] | undefined) =>
+ (ports ?? []).map(p => ({
+ name: p.name ?? null,
+ port: p.port,
+ targetPort: p.targetPort ?? null,
+ protocol: p.protocol ?? 'TCP',
+ }))
+ const desiredSpec = {
+ selector: headlessSvc.spec?.selector ?? null,
+ ports: normalizePorts(headlessSvc.spec?.ports),
+ }
+ const existingSpec = {
+ selector: existing.spec?.selector ?? null,
+ ports: normalizePorts(existing.spec?.ports),
+ }
+ if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {
+ return
+ }
+ const updatedSvc: k8s.V1Service = {
+ ...headlessSvc,
+ metadata: {
+ ...headlessSvc.metadata,
+ resourceVersion: existing.metadata?.resourceVersion,
+ },
+ spec: {
+ ...headlessSvc.spec,
+ clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,
+ },
+ }
+ await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
+ this.log.info(`Updated Headless Service "${name}"`)
+ }
+
+ /**
+ * Create-only headless Service: an existing Service is left as it is, and
+ * the read keeps the POST from running as an existence probe on every pass.
+ */
+ private async ensureHeadlessServiceExists(name: string, body: k8s.V1Service): Promise<void> {
+ const namespace = this.deps.config.sandboxNamespace
+ if (await this.readServiceIfExists(name, namespace)) return
+ await this.createIfNotExists(
+ () => this.deps.coreApi.createNamespacedService({ namespace, body }),
+ `Headless Service "${name}"`
+ )
+ }
+
+ private async readServiceIfExists(
+ name: string,
+ namespace: string
+ ): Promise<k8s.V1Service | undefined> {
+ try {
+ return await this.deps.coreApi.readNamespacedService({ name, namespace })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) return undefined
+ throw error
}
}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
index 09cd1bc53..fcfc16711 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts
@@ -17,7 +17,18 @@ import {
issueMcpHostRuntimeTokens,
issueMcpHostWorkflowControlToken,
} from '../../../src/workflow/mcpHostRuntimeTokenIssuerClient'
-import { buildMcpHostRouteAliasServiceName } from '../../../src/workflow/resourceNames'
+import {
+ buildArtifactReaderHeadlessService,
+ buildMcpHostHeadlessService,
+ buildMcpHostRouteAliasHeadlessService,
+ buildSnippetRunnerHeadlessService,
+} from '../../../src/workflow/podFactory'
+import {
+ buildArtifactReaderServiceName,
+ buildMcpHostRouteAliasServiceName,
+ buildMcpHostServiceName,
+ buildSnippetRunnerServiceName,
+} from '../../../src/workflow/resourceNames'
import {
type WorkflowRecipeSpec,
WorkflowReconciler,
@@ -486,6 +497,36 @@ function makeEncodedGfsAccess(
).toString('base64')
}
+type LiveService = ReturnType<typeof buildMcpHostHeadlessService>
+
+function asLiveService(service: LiveService, resourceVersion: string): LiveService {
+ return { ...service, metadata: { ...service.metadata, resourceVersion } }
+}
+
+/** Serve `readNamespacedService` from a fixed set of live Services; 404 for any other name. */
+function serveLiveServices(deps: WorkflowReconcilerDeps, live: LiveService[]): void {
+ const byKey = new Map(
+ live.map(service => [`${service.metadata?.namespace}/${service.metadata?.name}`, service])
+ )
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name, namespace }) => {
+ const service = byKey.get(`${namespace}/${name}`)
+ if (!service) throw { code: 404 }
+ return structuredClone(service)
+ })
+}
+
+function createdServiceNamesOf(deps: WorkflowReconcilerDeps): Array<string | undefined> {
+ return vi
+ .mocked(deps.coreApi.createNamespacedService)
+ .mock.calls.map(([arg]) => arg.body?.metadata?.name)
+}
+
+function serviceReadsOf(deps: WorkflowReconcilerDeps, name: string): number {
+ return vi
+ .mocked(deps.coreApi.readNamespacedService)
+ .mock.calls.filter(([arg]) => arg.name === name).length
+}
+
// ─── Tests ───────────────────────────────────────────────────────────────────
describe('WorkflowReconciler — reconcile loop', () => {
@@ -2910,6 +2951,190 @@ describe('WorkflowReconciler — reconcile loop', () => {
)
})
+ describe('headless Services are read before they are written', () => {
+ const namespace = 'sandbox-recipes'
+ const mcpHostName = buildMcpHostServiceName('test-wf')
+ const aliasName = buildMcpHostRouteAliasServiceName('test-wf', namespace)
+ const readerName = buildArtifactReaderServiceName('test-wf')
+ const snippetRunnerName = buildSnippetRunnerServiceName('test-wf')
+
+ const reconcileTriggeredRun = () =>
+ new WorkflowReconciler(deps).reconcile(
+ 'test-wf',
+ 'uid-123',
+ namespace,
+ makeSpec(),
+ undefined,
+ undefined,
+ undefined,
+ runId
+ )
+
+ it('sends neither POST nor PUT when the live mcp-host and route alias Services match', async () => {
+ serveLiveServices(deps, [
+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
+ asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
+ ])
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
+ name: mcpHostName,
+ namespace,
+ })
+ expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
+ name: aliasName,
+ namespace,
+ })
+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('reads, then creates the mcp-host Service exactly once when it is absent', async () => {
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {
+ const desired = buildMcpHostHeadlessService('test-wf', namespace)
+ const drifted = asLiveService(
+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
+ 'rv-7'
+ )
+ serveLiveServices(deps, [
+ drifted,
+ asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
+ ])
+
+ await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
+ const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
+ expect(replaceCalls).toHaveLength(1)
+ const [{ name, namespace: replacedIn, body }] = replaceCalls[0]
+ expect(name).toBe(mcpHostName)
+ expect(replacedIn).toBe(namespace)
+ expect(body.metadata?.resourceVersion).toBe('rv-7')
+ expect(body.spec?.clusterIP).toBe(drifted.spec?.clusterIP)
+ expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
+ })
+
+ it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {
+ const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')
+ let mcpHostReads = 0
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
+ if (name !== mcpHostName) throw { code: 404 }
+ mcpHostReads += 1
+ if (mcpHostReads === 1) throw { code: 404 }
+ return structuredClone(live)
+ })
+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
+ if (body.metadata?.name === mcpHostName) throw { code: 409 }
+ return body
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {
+ const forbidden = { code: 403 }
+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
+ if (name === mcpHostName) throw forbidden
+ throw { code: 404 }
+ })
+
+ const result = await reconcileTriggeredRun()
+
+ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
+ expect(result.workflowPhase).toBe('failed')
+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
+ })
+
+ describe('snippet-runner and artifact-reader Services', () => {
+ const reconcileSnippetWorkflow = () => {
+ vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({
+ metadata: { labels: { 'clerum.io/shared': 'true' } },
+ data: { key: 'dmFsdWU=' },
+ })
+ return new WorkflowReconciler(deps).reconcile(
+ 'test-wf',
+ 'uid-123',
+ namespace,
+ makeSpec({
+ agent: undefined,
+ workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
+ steps: [
+ {
+ id: 'snippet',
+ run: {
+ type: 'snippet',
+ language: 'typescript',
+ code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
+ capabilities: {
+ secrets: [
+ { alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } },
+ ],
+ postgres: { access: 'read', workloads: ['postgres'] },
+ },
+ },
+ },
+ ],
+ })
+ )
+ }
+
+ beforeEach(() => {
+ deps = makeDeps({
+ config: {
+ ...makeConfig(),
+ enableSnippetRuntime: true,
+ snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
+ } as never,
+ })
+ })
+
+ it('sends no POST for snippet-runner and artifact-reader Services that already exist', async () => {
+ serveLiveServices(deps, [
+ asLiveService(buildArtifactReaderHeadlessService('test-wf', namespace), 'rv-3'),
+ asLiveService(buildSnippetRunnerHeadlessService('test-wf', namespace), 'rv-4'),
+ ])
+
+ const result = await reconcileSnippetWorkflow()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, readerName)).toBe(1)
+ expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
+ expect(createdServiceNamesOf(deps)).not.toContain(readerName)
+ expect(createdServiceNamesOf(deps)).not.toContain(snippetRunnerName)
+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('reads, then creates snippet-runner and artifact-reader Services when they are absent', async () => {
+ const result = await reconcileSnippetWorkflow()
+
+ expect(result.workflowPhase).not.toBe('failed')
+ expect(serviceReadsOf(deps, readerName)).toBe(1)
+ expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
+ expect(createdServiceNamesOf(deps).filter(name => name === readerName)).toHaveLength(1)
+ expect(createdServiceNamesOf(deps).filter(name => name === snippetRunnerName)).toHaveLength(
+ 1
+ )
+ })
+ })
+ })
+
it('applies runtime NetworkPolicies before creating runtime Pods', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(