Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): read headless Services before writing them
Alfredo Lopez5c1e97b742ab
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 9dfd0c21a..68eac6fd9 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 774903918..cbb672a79 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.368",+ "version": "0.1.369","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.tsindex 04bad1016..02b9ab08f 100644--- a/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.codexScopeProvenance.test.ts@@ -151,7 +151,7 @@ function createHarness() {return {}}),readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),- readNamespacedService: vi.fn().mockResolvedValue({}),+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),readNamespacedConfigMap: vi.fn().mockResolvedValue(eligibleCodexConfigMap()),createNamespacedSecret: vi.fn().mockResolvedValue({}),diff --git a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.tsindex af4e19ab1..c59dc27c9 100644--- a/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.grokWrcFlag.test.ts@@ -90,7 +90,7 @@ function createHarness(grokSubscriptionEnabled: boolean, pluginWorkloadSdkEnablereturn {}}),readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),- readNamespacedService: vi.fn().mockResolvedValue({}),+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),readNamespacedConfigMap: vi.fn().mockResolvedValue(eligibleGrokConfigMap()),createNamespacedSecret: vi.fn().mockResolvedValue({}),diff --git a/workflow-recipes/src/workflow/workflowReconciler.test.ts b/workflow-recipes/src/workflow/workflowReconciler.test.tsindex f41ca9c17..3df82a3c1 100644--- a/workflow-recipes/src/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.test.ts@@ -116,7 +116,7 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =const mockCoreApi = {readNamespacedPod: vi.fn().mockResolvedValue({}),readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),- readNamespacedService: vi.fn().mockResolvedValue({}),+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedSecret: vi.fn().mockResolvedValue({}),createNamespacedConfigMap: vi.fn().mockResolvedValue({}),@@ -200,7 +200,9 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =return {}})mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })- mockCoreApi.readNamespacedService.mockResolvedValue({})+ // No Service exists until a test says so: reconcile reads before it+ // creates, so a resolved read would stand for a live (spec-less) Service.+ mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })mockCoreApi.createNamespacedSecret.mockResolvedValue({})mockCoreApi.createNamespacedConfigMap.mockResolvedValue({})mockCoreApi.createNamespacedService.mockResolvedValue({})@@ -483,6 +485,7 @@ describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () =})it('prefers artifact-reader HTTP artifact cleanup when the artifact-reader Service exists', async () => {+ mockCoreApi.readNamespacedService.mockResolvedValue({})const reconciler = new WorkflowReconciler(deps)const recipeName = 'agentic-recipe'diff --git a/workflow-recipes/src/workflow/workflowReconciler.ts b/workflow-recipes/src/workflow/workflowReconciler.tsindex b4df1587e..f123ebc26 100644--- a/workflow-recipes/src/workflow/workflowReconciler.ts+++ b/workflow-recipes/src/workflow/workflowReconciler.ts@@ -2096,45 +2096,21 @@ export class WorkflowReconciler {if (needsMcpHost && !awaitsTriggeredRun) {await this.ensureMcpHostHeadlessService(recipeName)- const routeAliasSvc = buildMcpHostRouteAliasHeadlessService(- recipeName,- this.deps.config.sandboxNamespace- )- await this.createIfNotExists(- () =>- this.deps.coreApi.createNamespacedService({- namespace: this.deps.config.sandboxNamespace,- body: routeAliasSvc,- }),- `Headless Service "${buildMcpHostRouteAliasServiceName(recipeName, this.deps.config.sandboxNamespace)}"`+ await this.ensureHeadlessServiceExists(+ buildMcpHostRouteAliasServiceName(recipeName, this.deps.config.sandboxNamespace),+ buildMcpHostRouteAliasHeadlessService(recipeName, this.deps.config.sandboxNamespace))}if (needsArtifactReader && !awaitsTriggeredRun) {- const readerSvc = buildArtifactReaderHeadlessService(- recipeName,- this.deps.config.sandboxNamespace- )- await this.createIfNotExists(- () =>- this.deps.coreApi.createNamespacedService({- namespace: this.deps.config.sandboxNamespace,- body: readerSvc,- }),- `Headless Service "${buildArtifactReaderServiceName(recipeName)}"`+ await this.ensureHeadlessServiceExists(+ buildArtifactReaderServiceName(recipeName),+ buildArtifactReaderHeadlessService(recipeName, this.deps.config.sandboxNamespace))}if (needsSnippetRunner && !awaitsTriggeredRun) {- const snippetRunnerSvc = buildSnippetRunnerHeadlessService(- recipeName,- this.deps.config.sandboxNamespace- )- await this.createIfNotExists(- () =>- this.deps.coreApi.createNamespacedService({- namespace: this.deps.config.sandboxNamespace,- body: snippetRunnerSvc,- }),- `Headless Service "${buildSnippetRunnerServiceName(recipeName)}"`+ await this.ensureHeadlessServiceExists(+ buildSnippetRunnerServiceName(recipeName),+ buildSnippetRunnerHeadlessService(recipeName, this.deps.config.sandboxNamespace))}@@ -3833,46 +3809,78 @@ export class WorkflowReconciler {const headlessSvc = buildMcpHostHeadlessService(recipeName, this.deps.config.sandboxNamespace)const namespace = this.deps.config.sandboxNamespaceconst name = buildMcpHostServiceName(recipeName)- try {- await this.deps.coreApi.createNamespacedService({ namespace, body: headlessSvc })- this.log.info(`Created Headless Service "${name}"`)- } catch (error: unknown) {- if (getErrorCode(error) !== 409) throw error- const existing = await this.deps.coreApi.readNamespacedService({ name, namespace })- // Skip the replace when the live Service already matches the desired- // spec — this method runs on every eager reconcile, and an unconditional- // GET+PUT churns resourceVersions and apiserver writes for no drift.- const normalizePorts = (ports: k8s.V1ServicePort[] | undefined) =>- (ports ?? []).map(p => ({- name: p.name ?? null,- port: p.port,- targetPort: p.targetPort ?? null,- protocol: p.protocol ?? 'TCP',- }))- const desiredSpec = {- selector: headlessSvc.spec?.selector ?? null,- ports: normalizePorts(headlessSvc.spec?.ports),- }- const existingSpec = {- selector: existing.spec?.selector ?? null,- ports: normalizePorts(existing.spec?.ports),- }- if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {+ // Read first: this method runs on every eager reconcile, and a POST used as+ // an existence probe is an apiserver write (409) on every pass.+ let existing = await this.readServiceIfExists(name, namespace)+ if (!existing) {+ try {+ await this.deps.coreApi.createNamespacedService({ namespace, body: headlessSvc })+ this.log.info(`Created Headless Service "${name}"`)return+ } catch (error: unknown) {+ if (getErrorCode(error) !== 409) throw error+ // Another writer created it between the read and the POST.+ existing = await this.deps.coreApi.readNamespacedService({ name, namespace })}- const updatedSvc: k8s.V1Service = {- ...headlessSvc,- metadata: {- ...headlessSvc.metadata,- resourceVersion: existing.metadata?.resourceVersion,- },- spec: {- ...headlessSvc.spec,- clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,- },- }- await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })- this.log.info(`Updated Headless Service "${name}"`)+ }+ // Skip the replace when the live Service already matches the desired+ // spec — an unconditional PUT churns resourceVersions and apiserver writes+ // for no drift.+ const normalizePorts = (ports: k8s.V1ServicePort[] | undefined) =>+ (ports ?? []).map(p => ({+ name: p.name ?? null,+ port: p.port,+ targetPort: p.targetPort ?? null,+ protocol: p.protocol ?? 'TCP',+ }))+ const desiredSpec = {+ selector: headlessSvc.spec?.selector ?? null,+ ports: normalizePorts(headlessSvc.spec?.ports),+ }+ const existingSpec = {+ selector: existing.spec?.selector ?? null,+ ports: normalizePorts(existing.spec?.ports),+ }+ if (JSON.stringify(desiredSpec) === JSON.stringify(existingSpec)) {+ return+ }+ const updatedSvc: k8s.V1Service = {+ ...headlessSvc,+ metadata: {+ ...headlessSvc.metadata,+ resourceVersion: existing.metadata?.resourceVersion,+ },+ spec: {+ ...headlessSvc.spec,+ clusterIP: existing.spec?.clusterIP ?? headlessSvc.spec?.clusterIP,+ },+ }+ await this.deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })+ this.log.info(`Updated Headless Service "${name}"`)+ }++ /**+ * Create-only headless Service: an existing Service is left as it is, and+ * the read keeps the POST from running as an existence probe on every pass.+ */+ private async ensureHeadlessServiceExists(name: string, body: k8s.V1Service): Promise<void> {+ const namespace = this.deps.config.sandboxNamespace+ if (await this.readServiceIfExists(name, namespace)) return+ await this.createIfNotExists(+ () => this.deps.coreApi.createNamespacedService({ namespace, body }),+ `Headless Service "${name}"`+ )+ }++ private async readServiceIfExists(+ name: string,+ namespace: string+ ): Promise<k8s.V1Service | undefined> {+ try {+ return await this.deps.coreApi.readNamespacedService({ name, namespace })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) return undefined+ throw error}}diff --git a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.tsindex 09cd1bc53..fcfc16711 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconciler.test.ts@@ -17,7 +17,18 @@ import {issueMcpHostRuntimeTokens,issueMcpHostWorkflowControlToken,} from '../../../src/workflow/mcpHostRuntimeTokenIssuerClient'-import { buildMcpHostRouteAliasServiceName } from '../../../src/workflow/resourceNames'+import {+ buildArtifactReaderHeadlessService,+ buildMcpHostHeadlessService,+ buildMcpHostRouteAliasHeadlessService,+ buildSnippetRunnerHeadlessService,+} from '../../../src/workflow/podFactory'+import {+ buildArtifactReaderServiceName,+ buildMcpHostRouteAliasServiceName,+ buildMcpHostServiceName,+ buildSnippetRunnerServiceName,+} from '../../../src/workflow/resourceNames'import {type WorkflowRecipeSpec,WorkflowReconciler,@@ -486,6 +497,36 @@ function makeEncodedGfsAccess().toString('base64')}+type LiveService = ReturnType<typeof buildMcpHostHeadlessService>++function asLiveService(service: LiveService, resourceVersion: string): LiveService {+ return { ...service, metadata: { ...service.metadata, resourceVersion } }+}++/** Serve `readNamespacedService` from a fixed set of live Services; 404 for any other name. */+function serveLiveServices(deps: WorkflowReconcilerDeps, live: LiveService[]): void {+ const byKey = new Map(+ live.map(service => [`${service.metadata?.namespace}/${service.metadata?.name}`, service])+ )+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name, namespace }) => {+ const service = byKey.get(`${namespace}/${name}`)+ if (!service) throw { code: 404 }+ return structuredClone(service)+ })+}++function createdServiceNamesOf(deps: WorkflowReconcilerDeps): Array<string | undefined> {+ return vi+ .mocked(deps.coreApi.createNamespacedService)+ .mock.calls.map(([arg]) => arg.body?.metadata?.name)+}++function serviceReadsOf(deps: WorkflowReconcilerDeps, name: string): number {+ return vi+ .mocked(deps.coreApi.readNamespacedService)+ .mock.calls.filter(([arg]) => arg.name === name).length+}+// ─── Tests ───────────────────────────────────────────────────────────────────describe('WorkflowReconciler — reconcile loop', () => {@@ -2910,6 +2951,190 @@ describe('WorkflowReconciler — reconcile loop', () => {)})+ describe('headless Services are read before they are written', () => {+ const namespace = 'sandbox-recipes'+ const mcpHostName = buildMcpHostServiceName('test-wf')+ const aliasName = buildMcpHostRouteAliasServiceName('test-wf', namespace)+ const readerName = buildArtifactReaderServiceName('test-wf')+ const snippetRunnerName = buildSnippetRunnerServiceName('test-wf')++ const reconcileTriggeredRun = () =>+ new WorkflowReconciler(deps).reconcile(+ 'test-wf',+ 'uid-123',+ namespace,+ makeSpec(),+ undefined,+ undefined,+ undefined,+ runId+ )++ it('sends neither POST nor PUT when the live mcp-host and route alias Services match', async () => {+ serveLiveServices(deps, [+ asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),+ asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),+ ])++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({+ name: mcpHostName,+ namespace,+ })+ expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({+ name: aliasName,+ namespace,+ })+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)+ expect(createdServiceNamesOf(deps)).not.toContain(aliasName)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('reads, then creates the mcp-host Service exactly once when it is absent', async () => {+ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {+ const desired = buildMcpHostHeadlessService('test-wf', namespace)+ const drifted = asLiveService(+ { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },+ 'rv-7'+ )+ serveLiveServices(deps, [+ drifted,+ asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),+ ])++ await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)+ const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls+ expect(replaceCalls).toHaveLength(1)+ const [{ name, namespace: replacedIn, body }] = replaceCalls[0]+ expect(name).toBe(mcpHostName)+ expect(replacedIn).toBe(namespace)+ expect(body.metadata?.resourceVersion).toBe('rv-7')+ expect(body.spec?.clusterIP).toBe(drifted.spec?.clusterIP)+ expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])+ })++ it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {+ const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')+ let mcpHostReads = 0+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {+ if (name !== mcpHostName) throw { code: 404 }+ mcpHostReads += 1+ if (mcpHostReads === 1) throw { code: 404 }+ return structuredClone(live)+ })+ vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {+ if (body.metadata?.name === mcpHostName) throw { code: 409 }+ return body+ })++ const result = await reconcileTriggeredRun()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, mcpHostName)).toBe(2)+ expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {+ const forbidden = { code: 403 }+ vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {+ if (name === mcpHostName) throw forbidden+ throw { code: 404 }+ })++ const result = await reconcileTriggeredRun()++ expect(serviceReadsOf(deps, mcpHostName)).toBe(1)+ expect(result.workflowPhase).toBe('failed')+ expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)+ })++ describe('snippet-runner and artifact-reader Services', () => {+ const reconcileSnippetWorkflow = () => {+ vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({+ metadata: { labels: { 'clerum.io/shared': 'true' } },+ data: { key: 'dmFsdWU=' },+ })+ return new WorkflowReconciler(deps).reconcile(+ 'test-wf',+ 'uid-123',+ namespace,+ makeSpec({+ agent: undefined,+ workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],+ steps: [+ {+ id: 'snippet',+ run: {+ type: 'snippet',+ language: 'typescript',+ code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',+ capabilities: {+ secrets: [+ { alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } },+ ],+ postgres: { access: 'read', workloads: ['postgres'] },+ },+ },+ },+ ],+ })+ )+ }++ beforeEach(() => {+ deps = makeDeps({+ config: {+ ...makeConfig(),+ enableSnippetRuntime: true,+ snippetRunnerImage: 'clerum/workflow-snippet-runner:test',+ } as never,+ })+ })++ it('sends no POST for snippet-runner and artifact-reader Services that already exist', async () => {+ serveLiveServices(deps, [+ asLiveService(buildArtifactReaderHeadlessService('test-wf', namespace), 'rv-3'),+ asLiveService(buildSnippetRunnerHeadlessService('test-wf', namespace), 'rv-4'),+ ])++ const result = await reconcileSnippetWorkflow()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, readerName)).toBe(1)+ expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)+ expect(createdServiceNamesOf(deps)).not.toContain(readerName)+ expect(createdServiceNamesOf(deps)).not.toContain(snippetRunnerName)+ expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('reads, then creates snippet-runner and artifact-reader Services when they are absent', async () => {+ const result = await reconcileSnippetWorkflow()++ expect(result.workflowPhase).not.toBe('failed')+ expect(serviceReadsOf(deps, readerName)).toBe(1)+ expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)+ expect(createdServiceNamesOf(deps).filter(name => name === readerName)).toHaveLength(1)+ expect(createdServiceNamesOf(deps).filter(name => name === snippetRunnerName)).toHaveLength(+ 1+ )+ })+ })+ })+it('applies runtime NetworkPolicies before creating runtime Pods', async () => {const reconciler = new WorkflowReconciler(deps)await reconciler.reconcile(