Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(workflow-recipes): back off policy-pending requeues and read before apply
Alfredo Lopeze713760866dd
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 2104b456b..f2d9bb5a5 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.364",+ "version": "0.1.365","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.364",+ "version": "0.1.365","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 334b84f31..df23a6dda 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.364",+ "version": "0.1.365","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex a49b9a9c7..2ae4f152d 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -5,6 +5,7 @@*/import { describe, expect, it, vi } from 'vitest'import { WorkflowRecipeCRD } from '../types'+import { captureLogger } from './__tests__/captureLogger'import {DelegationDeps,PRE_DEPLOY_ANNOTATION,@@ -1885,6 +1886,7 @@ describe('delegateTransportWorkloads', () => {.mockRejectedValueOnce({ code: 404 }),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -1942,7 +1944,10 @@ describe('delegateTransportWorkloads', () => {),replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }+ const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ createNamespacedService: vi.fn().mockResolvedValue({}),+ }const deps: DelegationDeps = {customApi: mockCustomApi as unknown as DelegationDeps['customApi'],coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],@@ -2017,7 +2022,10 @@ describe('delegateTransportWorkloads', () => {),replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }+ const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ createNamespacedService: vi.fn().mockResolvedValue({}),+ }const deps: DelegationDeps = {customApi: mockCustomApi as unknown as DelegationDeps['customApi'],coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],@@ -2063,7 +2071,10 @@ describe('delegateTransportWorkloads', () => {}),replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }+ const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ createNamespacedService: vi.fn().mockResolvedValue({}),+ }const deps: DelegationDeps = {customApi: mockCustomApi as unknown as DelegationDeps['customApi'],coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],@@ -2113,7 +2124,10 @@ describe('delegateTransportWorkloads', () => {}),replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }+ const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),+ createNamespacedService: vi.fn().mockResolvedValue({}),+ }const deps: DelegationDeps = {customApi: mockCustomApi as unknown as DelegationDeps['customApi'],coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],@@ -2157,6 +2171,7 @@ describe('delegateTransportWorkloads', () => {patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2175,6 +2190,7 @@ describe('delegateTransportWorkloads', () => {createNamespacedCustomObject: vi.fn().mockResolvedValue({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2208,6 +2224,7 @@ describe('delegateTransportWorkloads', () => {.mockRejectedValueOnce({ code: 404 }),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2236,6 +2253,7 @@ describe('delegateTransportWorkloads', () => {replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2263,7 +2281,7 @@ describe('delegateTransportWorkloads', () => {})it('rejects an existing McpServer owned by a different recipe label', async () => {- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const errorLog = captureLogger('error')const mockCustomApi = {createNamespacedCustomObject: vi.fn().mockResolvedValue({}),getNamespacedCustomObject: vi.fn().mockResolvedValueOnce({@@ -2272,6 +2290,7 @@ describe('delegateTransportWorkloads', () => {replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2283,12 +2302,20 @@ describe('delegateTransportWorkloads', () => {await expect(delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())).rejects.toThrow(/Delegation failed for workload\(s\): redis-mcp/)+ // Witness: the aggregate rejection came from the ownership guard, not from+ // any other failure on the way to it.+ expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {+ workloadId: 'redis-mcp',+ error: expect.objectContaining({+ message: expect.stringContaining('already exists for recipe "other-recipe"'),+ }),+ })expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalledWith(expect.objectContaining({ plural: 'contexts' }))} finally {- errorSpy.mockRestore()+ errorLog.mockRestore()}})@@ -2313,6 +2340,7 @@ describe('delegateTransportWorkloads', () => {.mockResolvedValueOnce({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2336,7 +2364,7 @@ describe('delegateTransportWorkloads', () => {})it('keeps delegation non-fatal when a recipe-owned McpServer remains after conflict retries', async () => {- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})+ const warnLog = captureLogger('warn')const recipe = makeRecipe()const desiredManifest = buildMcpServerManifest(recipe.spec.workloads![0], recipe, 'mcp-server')!const mockCustomApi = {@@ -2363,6 +2391,7 @@ describe('delegateTransportWorkloads', () => {.mockRejectedValueOnce({ code: 409 }),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2375,17 +2404,20 @@ describe('delegateTransportWorkloads', () => {expect(delegated).toEqual(['test-recipe-redis-mcp'])expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('existing spec already matches'))+ expect(warnLog).toHaveBeenCalledWith(+ expect.stringContaining('existing spec already matches'),+ { mcpServer: 'test-recipe-redis-mcp', retries: 3 }+ )expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(expect.objectContaining({ plural: 'contexts' }))} finally {- warnSpy.mockRestore()+ warnLog.mockRestore()}})it('does not hide exhausted McpServer conflicts when the remaining recipe-owned object is stale', async () => {- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})+ const errorLog = captureLogger('error')const recipe = makeRecipe()const desiredManifest = buildMcpServerManifest(recipe.spec.workloads![0], recipe, 'mcp-server')!const staleSpec = { ...(desiredManifest.spec as Record<string, unknown>), image: 'old:image' }@@ -2413,6 +2445,7 @@ describe('delegateTransportWorkloads', () => {.mockRejectedValueOnce({ code: 409 }),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2424,11 +2457,20 @@ describe('delegateTransportWorkloads', () => {await expect(delegateTransportWorkloads(deps, recipe, 'mcp-server', new Map())).rejects.toThrow(/Delegation failed for workload\(s\): redis-mcp/)+ // Witness: the rejection is the exhausted-conflict error, after all three+ // replace attempts, not an earlier failure.+ expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)+ expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {+ workloadId: 'redis-mcp',+ error: expect.objectContaining({+ message: expect.stringContaining('failed to update after conflict retries'),+ }),+ })expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalledWith(expect.objectContaining({ plural: 'contexts' }))} finally {- errorSpy.mockRestore()+ errorLog.mockRestore()}})@@ -2447,6 +2489,7 @@ describe('delegateTransportWorkloads', () => {replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2509,6 +2552,7 @@ describe('delegateTransportWorkloads', () => {.mockRejectedValueOnce({ code: 404 }),}const mockCoreApi = {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),}const deps: DelegationDeps = {@@ -2742,7 +2786,7 @@ describe('waitForExternalEgressReady', () => {}await expect(- waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)+ waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 'test-recipe', 1000)).resolves.toEqual({ ready: true, pending: [], failed: [] })})@@ -2766,7 +2810,13 @@ describe('waitForExternalEgressReady', () => {coreApi: {} as any,}- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)+ const result = await waitForExternalEgressReady(+ deps,+ ['web-search'],+ 'mcp-server',+ 'test-recipe',+ 1+ )expect(result.ready).toBe(false)expect(result.pending).toEqual(['web-search'])@@ -2794,7 +2844,13 @@ describe('waitForExternalEgressReady', () => {coreApi: {} as any,}- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)+ const result = await waitForExternalEgressReady(+ deps,+ ['web-search'],+ 'mcp-server',+ 'test-recipe',+ 1+ )expect(result.ready).toBe(false)expect(result.pending).toEqual(['web-search'])@@ -2822,7 +2878,13 @@ describe('waitForExternalEgressReady', () => {coreApi: {} as any,}- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)+ const result = await waitForExternalEgressReady(+ deps,+ ['web-search'],+ 'mcp-server',+ 'test-recipe',+ 1+ )expect(result).toEqual({ ready: false, pending: ['web-search'], failed: [] })})@@ -2847,7 +2909,13 @@ describe('waitForExternalEgressReady', () => {coreApi: {} as any,}- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)+ const result = await waitForExternalEgressReady(+ deps,+ ['web-search'],+ 'mcp-server',+ 'test-recipe',+ 1000+ )expect(result).toEqual({ready: false,pending: ['web-search'],@@ -2878,7 +2946,7 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {coreApi: {} as any,}- const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)+ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)expect(result.ready).toBe(false)expect(result.pending).toEqual(['srv-a'])@@ -2897,7 +2965,7 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {coreApi: {} as any,}- const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)+ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)expect(result.ready).toBe(false)expect(result.pending).toEqual(['srv-a'])@@ -2919,7 +2987,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {coreApi: {} as any,}- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ await expect(+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)+ ).resolves.toEqual({ready: true,pending: [],})@@ -2935,7 +3005,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {coreApi: {} as any,}- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ await expect(+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)+ ).resolves.toEqual({ready: true,pending: [],})@@ -2949,7 +3021,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {coreApi: {} as any,}- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ await expect(+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)+ ).resolves.toEqual({ready: true,pending: [],})@@ -2996,6 +3070,7 @@ describe('Issue #637 — transport Secret ownership gate', () => {...overrides?.customApi,} as unknown as DelegationDeps['customApi'],coreApi: {+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),createNamespacedService: vi.fn().mockResolvedValue({}),deleteNamespacedService: vi.fn().mockResolvedValue({}),...overrides?.coreApi,@@ -3140,3 +3215,119 @@ describe('Issue #637 — transport Secret ownership gate', () => {})})})++// ─── Transport Service read-first apply ──────────────────────────────────────++describe('transport Service read-first apply', () => {+ type LiveService = {+ metadata: { name: string; resourceVersion?: string; annotations?: Record<string, string> }+ spec: { clusterIP?: string; ports?: Array<{ port: number }> }+ }+ const NS = 'mcp-server'+ const NAME = 'test-recipe-redis-mcp'+ const LIVE_CLUSTER_IP = '10.0.0.9'++ // Stateful Service double: an absent object reads as 404, a create of a+ // present one is a 409, the apiserver assigns clusterIP on create, and a+ // replace must carry the live resourceVersion.+ function serviceDeps() {+ const live = new Map<string, LiveService>()+ const coreApi = {+ readNamespacedService: vi.fn(async ({ name }: { name: string }) => {+ const found = live.get(name)+ if (!found) throw { code: 404 }+ return structuredClone(found)+ }),+ createNamespacedService: vi.fn(async ({ body }: { body: LiveService }) => {+ if (live.has(body.metadata.name)) throw { code: 409 }+ const stored = structuredClone(body)+ stored.metadata.resourceVersion = '1'+ stored.spec.clusterIP = LIVE_CLUSTER_IP+ live.set(body.metadata.name, stored)+ return {}+ }),+ replaceNamespacedService: vi.fn(+ async ({ name, body }: { name: string; body: LiveService }) => {+ const current = live.get(name)+ if (!current || body.metadata.resourceVersion !== current.metadata.resourceVersion) {+ throw { code: 409 }+ }+ const stored = structuredClone(body)+ stored.metadata.resourceVersion = String(Number(current.metadata.resourceVersion) + 1)+ live.set(name, stored)+ return {}+ }+ ),+ }+ const deps: DelegationDeps = {+ customApi: {+ createNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),+ } as unknown as DelegationDeps['customApi'],+ coreApi: coreApi as unknown as DelegationDeps['coreApi'],+ }+ const clear = () => {+ coreApi.readNamespacedService.mockClear()+ coreApi.createNamespacedService.mockClear()+ coreApi.replaceNamespacedService.mockClear()+ }+ return { deps, coreApi, clear, live: () => live.get(NAME) }+ }++ it('reads an absent transport Service first, then sends exactly one POST and no PUT', async () => {+ const { deps, coreApi } = serviceDeps()++ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())++ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)+ expect(coreApi.readNamespacedService.mock.invocationCallOrder[0]).toBeLessThan(+ coreApi.createNamespacedService.mock.invocationCallOrder[0]+ )+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('sends neither POST nor PUT for an unchanged transport Service', async () => {+ const { deps, coreApi, clear } = serviceDeps()+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())+ clear()++ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())++ // Witness: the gate read the live Service before deciding not to write.+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(1)+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ it('replaces a changed transport Service once, keeping the live clusterIP', async () => {+ const { deps, coreApi, clear, live } = serviceDeps()+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())+ clear()+ const changed = makeRecipe()+ changed.spec.workloads![0].port = 3100++ await delegateTransportWorkloads(deps, changed, NS, new Map())++ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)+ expect(live()!.spec.clusterIP).toBe(LIVE_CLUSTER_IP)+ expect(live()!.spec.ports![0].port).toBe(3100)+ })++ it('skips the write when another writer created the desired Service between read and POST', async () => {+ const { deps, coreApi, clear } = serviceDeps()+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())+ clear()+ // The gate read misses the object the other writer is about to create.+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })++ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())++ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)+ // Witness: the conflict re-read the winner before deciding not to write.+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })+})diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex 305370cba..efc6e9352 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -21,6 +21,7 @@ import {} from '@clerum/workflow-runtime-core'import { loadConfig } from '../config'import { createLogger } from '../observability/logger'+import type { Logger } from '../observability/logger'import { WorkflowRecipeCRD, WorkloadDef } from '../types'import { CRD_GROUP, CRD_VERSION } from './crdConstants'import { getErrorCode } from './k8sErrors'@@ -581,30 +582,55 @@ export function buildTransportService(// ─── K8s API Operations ──────────────────────────────────────────────-/** Create or update a ClusterIP Service. Idempotent (409 → read + replace preserving clusterIP). */+/**+ * Create or update a ClusterIP Service, reading first. The desired manifest is+ * stamped with its spec hash; a live Service carrying the same hash is left+ * alone, so a steady-state reconcile sends neither POST nor PUT. An absent+ * Service is created; a 409 on that create means another writer got there+ * between the read and the POST, so the gate is re-evaluated against the object+ * it wrote. A changed Service is replaced carrying the live resourceVersion and+ * clusterIP (spec.clusterIP is immutable once assigned).+ */async function ensureTransportService(deps: DelegationDeps,svc: k8s.V1Service,- namespace: string+ namespace: string,+ log: Logger): Promise<void> {const name = svc.metadata!.name!+ stampSpecHash(svc)++ let existing: k8s.V1Service | nulltry {- await deps.coreApi.createNamespacedService({ namespace, body: svc })- console.log(`[MCP-Delegation] Created transport Service "${name}"`)+ existing = await deps.coreApi.readNamespacedService({ name, namespace })} catch (error) {- if (getErrorCode(error) === 409) {- const existing = await deps.coreApi.readNamespacedService({ name, namespace })- const updatedSvc: k8s.V1Service = {- ...svc,- metadata: { ...svc.metadata, resourceVersion: existing.metadata?.resourceVersion },- spec: { ...svc.spec, clusterIP: existing.spec?.clusterIP },- }- await deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })- console.log(`[MCP-Delegation] Updated transport Service "${name}"`)- } else {- throw error+ if (getErrorCode(error) !== 404) throw error+ existing = null+ }++ if (!existing) {+ try {+ await deps.coreApi.createNamespacedService({ namespace, body: svc })+ log.info('Created transport Service', { service: name })+ return+ } catch (error) {+ if (getErrorCode(error) !== 409) throw error}+ existing = await deps.coreApi.readNamespacedService({ name, namespace })}++ if (specHashUnchanged(svc, existing)) {+ log.debug('Transport Service unchanged; skipping update', { service: name })+ return+ }++ const updatedSvc: k8s.V1Service = {+ ...svc,+ metadata: { ...svc.metadata, resourceVersion: existing.metadata?.resourceVersion },+ spec: { ...svc.spec, clusterIP: existing.spec?.clusterIP },+ }+ await deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })+ log.info('Updated transport Service', { service: name })}/** Create or update an McpServer CRD. Idempotent (409 → replace).@@ -616,7 +642,8 @@ async function ensureTransportService(async function ensureMcpServer(deps: DelegationDeps,manifest: Record<string, unknown>,- namespace: string+ namespace: string,+ log: Logger): Promise<void> {const meta = manifest.metadata as { name: string; labels?: Record<string, string> }const desiredRecipeLabel = meta.labels?.['clerum.io/recipe']@@ -753,9 +780,10 @@ async function ensureMcpServer(isRecipeOwned === desiredRecipeLabel &&mcpServerSpecMatches(latest.spec, manifest.spec)) {- console.warn(- `[MCP-Delegation] McpServer "${meta.name}" kept existing recipe-owned object after ` +- `${MCPSERVER_REPLACE_CONFLICT_RETRIES} replace conflict retries; existing spec already matches desired state.`+ log.warn(+ 'McpServer kept existing recipe-owned object after replace conflict retries; ' ++ 'existing spec already matches desired state',+ { mcpServer: meta.name, retries: MCPSERVER_REPLACE_CONFLICT_RETRIES })return false}@@ -774,9 +802,9 @@ async function ensureMcpServer(const existing = await readExisting()const replaced = await replaceFromExisting(existing)if (replaced) {- console.log(`[MCP-Delegation] Updated McpServer "${meta.name}"`)+ log.info('Updated McpServer', { mcpServer: meta.name })} else {- console.log(`[MCP-Delegation] McpServer "${meta.name}" unchanged; skipping update`)+ log.info('McpServer unchanged; skipping update', { mcpServer: meta.name })}} catch (error) {if (getErrorCode(error) === 404) {@@ -789,17 +817,17 @@ async function ensureMcpServer(plural: MCPSERVER_PLURAL,body: manifest,})- console.log(`[MCP-Delegation] Created McpServer "${meta.name}"`)+ log.info('Created McpServer', { mcpServer: meta.name })} catch (createError) {if (getErrorCode(createError) === 409) {const latest = await readExisting()const replaced = await replaceFromExisting(latest)if (replaced) {- console.log(`[MCP-Delegation] Updated McpServer "${meta.name}" after create conflict`)+ log.info('Updated McpServer after create conflict', { mcpServer: meta.name })} else {- console.log(- `[MCP-Delegation] McpServer "${meta.name}" unchanged after create conflict; skipping update`- )+ log.info('McpServer unchanged after create conflict; skipping update', {+ mcpServer: meta.name,+ })}} else {throw createError@@ -817,7 +845,8 @@ async function ensureMcpServer(async function deleteMcpServer(deps: DelegationDeps,name: string,- namespace: string+ namespace: string,+ log: Logger): Promise<void> {try {await deps.customApi.deleteNamespacedCustomObject({@@ -827,10 +856,10 @@ async function deleteMcpServer(plural: MCPSERVER_PLURAL,name,})- console.log(`[MCP-Delegation] Deleted McpServer "${name}"`)+ log.info('Deleted McpServer', { mcpServer: name })} catch (error) {if (getErrorCode(error) === 404) {- console.log(`[MCP-Delegation] McpServer "${name}" already gone`)+ log.info('McpServer already gone', { mcpServer: name })} else {throw error}@@ -854,10 +883,11 @@ export async function deleteTransportDelegation(): Promise<void> {if (!workload.transport) returnconst name = mcpServerName(recipe.metadata.name, workload.id, recipe)- await deleteMcpServer(deps, name, namespace)+ const log = createLogger('wrc', recipe.metadata.name)+ await deleteMcpServer(deps, name, namespace, log)try {await deps.coreApi.deleteNamespacedService({ name, namespace })- console.log(`[MCP-Delegation] Deleted transport Service "${name}" (ownership revoked)`)+ log.info('Deleted transport Service (ownership revoked)', { service: name })} catch (error) {if (getErrorCode(error) !== 404) throw error}@@ -1006,10 +1036,7 @@ export async function ensureRecipeContext(throw new Error(`failed to update Context "${contextName}" after conflict retries`)}- // One mechanism for all three outcomes of this one decision. The skip branch- // introduced `createLogger` while its siblings stayed on `console.log`, which- // left the same event reported two ways (#568 review, R1-L5). Scoped to this- // function: the file-wide console -> logger migration is #569.+ // One mechanism for all three outcomes of this one decision (#568 review, R1-L5).const log = createLogger('wrc', recipeName)try {@@ -1087,14 +1114,15 @@ export async function preDeployMcpServers(namespace: string,secretAccess: ReadonlyMap<string, SecretAccess>): Promise<string[]> {+ const log = createLogger('wrc', recipe.metadata.name)const transportWorkloads = (recipe.spec.workloads ?? []).filter(w => !!w.transport).filter(workload => {if (transportWorkloadSecretDenied(workload, secretAccess)) {- console.error(- `[MCP-Delegation] Issue #637: refusing to pre-deploy transport workload "${workload.id}" ` +- `in recipe "${recipe.metadata.name}" — it references a Secret it does not own ` +- `(denied/unverified). No McpServer CRD or Service will be created.`+ log.error(+ 'Issue #637: refusing to pre-deploy transport workload that references a Secret ' ++ 'it does not own (denied/unverified). No McpServer CRD or Service will be created.',+ { workloadId: workload.id })return false}@@ -1117,10 +1145,10 @@ export async function preDeployMcpServers(// Also create the transport Service (needed for DNS resolution in McpServer URL)const svc = buildTransportService(workload, recipe, namespace)if (svc) {- await ensureTransportService(deps, svc, namespace)+ await ensureTransportService(deps, svc, namespace, log)}- await ensureMcpServer(deps, manifest, namespace)+ await ensureMcpServer(deps, manifest, namespace, log)return mcpServerName(recipe.metadata.name, workload.id, recipe)}))@@ -1132,9 +1160,7 @@ export async function preDeployMcpServers(preDeployed.push(r.value)} else if (r.status === 'rejected') {const workloadId = transportWorkloads[i].id- console.warn(- `[MCP-Delegation] Pre-deploy failed for workload "${workloadId}": ${String(r.reason)}`- )+ log.warn('Pre-deploy failed for workload', { workloadId, error: r.reason })errors.push({ workloadId, error: r.reason })}}@@ -1195,10 +1221,12 @@ export async function waitForNetworkReady(deps: DelegationDeps,serverNames: string[],namespace: string,+ recipeName: string,timeoutMs: number = NETWORK_READY_TIMEOUT_MS): Promise<{ ready: boolean; pending: string[] }> {if (serverNames.length === 0) return { ready: true, pending: [] }+ const log = createLogger('wrc', recipeName)const deadline = Date.now() + timeoutMsconst pending = new Set(serverNames)@@ -1238,7 +1266,7 @@ export async function waitForNetworkReady(for (const { name, ready } of results) {if (ready) {pending.delete(name)- console.log(`[MCP-Delegation] Network ready for "${name}"`)+ log.info('Network ready', { mcpServer: name })}}@@ -1248,9 +1276,10 @@ export async function waitForNetworkReady(}if (pending.size > 0) {- console.warn(- `[MCP-Delegation] Network ready timeout (${timeoutMs}ms) for: ${[...pending].join(', ')}. ` +- `This generic readiness check is not used to unblock workloads with external egress.`+ log.warn(+ 'Network ready timeout. This generic readiness check is not used to unblock ' ++ 'workloads with external egress.',+ { timeoutMs, pending: [...pending].join(', ') })}@@ -1277,6 +1306,7 @@ export async function waitForExternalEgressReady(deps: DelegationDeps,serverNames: string[],namespace: string,+ recipeName: string,timeoutMs: number = NETWORK_READY_TIMEOUT_MS): Promise<{ready: boolean@@ -1285,6 +1315,7 @@ export async function waitForExternalEgressReady(}> {if (serverNames.length === 0) return { ready: true, pending: [], failed: [] }+ const log = createLogger('wrc', recipeName)const deadline = Date.now() + timeoutMsconst pending = new Set(serverNames)const failed = new Map<string, string>()@@ -1335,7 +1366,7 @@ export async function waitForExternalEgressReady(for (const result of results) {if (result.status === 'True') {pending.delete(result.name)- console.log(`[MCP-Delegation] External egress ready for "${result.name}"`)+ log.info('External egress ready', { mcpServer: result.name })} else if (result.status === 'False') {if (!result.reason || !RETRYABLE_EXTERNAL_EGRESS_REASONS.has(result.reason)) {failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)@@ -1350,12 +1381,9 @@ export async function waitForExternalEgressReady(const failedList = [...failed.entries()].map(([name, message]) => ({ name, message }))if (pending.size > 0 || failedList.length > 0) {- console.warn(- `[MCP-Delegation] External egress readiness not achieved for: ${[- ...pending,- ...failed.keys(),- ].join(', ')}`- )+ log.warn('External egress readiness not achieved', {+ pending: [...pending, ...failed.keys()].join(', '),+ })}return {@@ -1400,14 +1428,15 @@ export async function delegateTransportWorkloads(namespace: string,secretAccess: ReadonlyMap<string, SecretAccess>): Promise<string[]> {+ const log = createLogger('wrc', recipe.metadata.name)const transportWorkloads = (recipe.spec.workloads ?? []).filter(w => !!w.transport).filter(workload => {if (transportWorkloadSecretDenied(workload, secretAccess)) {- console.error(- `[MCP-Delegation] Issue #637: refusing to delegate transport workload "${workload.id}" ` +- `in recipe "${recipe.metadata.name}" — it references a Secret it does not own ` +- `(denied/unverified). Its McpServer CRD will not be (re)created.`+ log.error(+ 'Issue #637: refusing to delegate transport workload that references a Secret ' ++ 'it does not own (denied/unverified). Its McpServer CRD will not be (re)created.',+ { workloadId: workload.id })return false}@@ -1422,13 +1451,13 @@ export async function delegateTransportWorkloads(// 1. Create transport Serviceconst svc = buildTransportService(workload, recipe, namespace)if (svc) {- await ensureTransportService(deps, svc, namespace)+ await ensureTransportService(deps, svc, namespace, log)}// 2. Create McpServer CRDconst manifest = buildMcpServerManifest(workload, recipe, namespace, secretAccess)if (manifest) {- await ensureMcpServer(deps, manifest, namespace)+ await ensureMcpServer(deps, manifest, namespace, log)}return serverName@@ -1442,7 +1471,7 @@ export async function delegateTransportWorkloads(delegated.push(r.value)} else {const workloadId = transportWorkloads[i].id- console.error(`[MCP-Delegation] Failed to delegate workload "${workloadId}":`, r.reason)+ log.error('Failed to delegate workload', { workloadId, error: r.reason })errors.push({ workloadId, error: r.reason })}}@@ -1497,6 +1526,7 @@ export async function cleanupDelegation(recipe: WorkflowRecipeCRD,namespace: string): Promise<void> {+ const log = createLogger('wrc', recipe.metadata.name)const transportWorkloads = (recipe.spec.workloads ?? []).filter(w => !!w.transport)const cleanupFailures: string[] = []@@ -1506,15 +1536,15 @@ export async function cleanupDelegation(const serverName = mcpServerName(recipe.metadata.name, workload.id, recipe)// 1. Delete McpServer CRD (ownerRef GC also handles this)- await deleteMcpServer(deps, serverName, namespace)+ await deleteMcpServer(deps, serverName, namespace, log)// 2. Delete transport Service (belt-and-suspenders)try {await deps.coreApi.deleteNamespacedService({ name: serverName, namespace })- console.log(`[MCP-Delegation] Deleted transport Service "${serverName}"`)+ log.info('Deleted transport Service', { service: serverName })} catch (error) {if (getErrorCode(error) === 404) {- console.log(`[MCP-Delegation] Transport Service "${serverName}" already gone`)+ log.info('Transport Service already gone', { service: serverName })} else {throw error}@@ -1577,10 +1607,10 @@ export async function cleanupDelegation(if (!removed) {throw new Error(`failed to update Context "${recipeContextName}" after conflict retries`)}- console.log(`[MCP-Delegation] Removed recipe servers from Context "${recipeContextName}"`)+ log.info('Removed recipe servers from Context', { contextName: recipeContextName })} catch (error) {if (getErrorCode(error) === 404) {- console.log(`[MCP-Delegation] Explicit Context "${recipeContextName}" already gone`)+ log.info('Explicit Context already gone', { contextName: recipeContextName })} else {cleanupFailures.push(`Context "${recipeContextName}": ${@@ -1598,10 +1628,10 @@ export async function cleanupDelegation(plural: CONTEXT_PLURAL,name: recipeContextName,})- .then(() => console.log(`[MCP-Delegation] Deleted per-recipe Context "${recipeContextName}"`))+ .then(() => log.info('Deleted per-recipe Context', { contextName: recipeContextName })).catch(error => {if (getErrorCode(error) === 404) {- console.log(`[MCP-Delegation] Per-recipe Context "${recipeContextName}" already gone`)+ log.info('Per-recipe Context already gone', { contextName: recipeContextName })} else {cleanupFailures.push(`Context "${recipeContextName}": ${diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 282666034..c5b9c87e0 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -14,7 +14,10 @@ import { defaultFqdnLookup } from './fqdnResolver'import { isRetryableInfraError } from './k8sErrors'import type { NetworkPolicyFamily } from './networkPolicyConvergence'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'-import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'+import {+ PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,+ PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,+} from './pluginWorkloadSdkValidator'import {buildUiIngressNetworkPolicy,buildWorkloadIngressNetworkPolicy,@@ -208,6 +211,121 @@ function egressStateAnnotations(}}+type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & Record<+ string,+ unknown+>++/**+ * Stateful apiserver double for one spec-hash-gated kind. `createOrReplace`+ * reads before it writes, so an object this test never created must read as a+ * 404; one it created or replaced reads back as written, carrying `status` —+ * the same "ready" status the observation path used to get from a static read.+ * `admit` models fields the apiserver assigns on create (a Service's clusterIP).+ */+function installLiveStore(+ api: {+ read: ReturnType<typeof vi.fn>+ create: ReturnType<typeof vi.fn>+ replace: ReturnType<typeof vi.fn>+ },+ status: Record<string, unknown>,+ admit?: (stored: LiveManifest) => void+): void {+ const live = new Map<string, LiveManifest>()+ const key = (namespace: string, name: string) => `${namespace}/${name}`+ api.read+ .mockReset()+ .mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => {+ const found = live.get(key(namespace, name))+ if (!found) throw { code: 404 }+ return structuredClone(found)+ })+ api.create+ .mockReset()+ .mockImplementation(async ({ namespace, body }: { namespace: string; body: LiveManifest }) => {+ const k = key(namespace, body.metadata.name)+ if (live.has(k)) throw { code: 409 }+ const stored = structuredClone(body)+ stored.metadata = { ...stored.metadata, resourceVersion: '1', generation: 1 }+ admit?.(stored)+ live.set(k, { ...stored, status: structuredClone(status) })+ return {}+ })+ api.replace+ .mockReset()+ .mockImplementation(+ async ({+ name,+ namespace,+ body,+ }: {+ name: string+ namespace: string+ body: LiveManifest+ }) => {+ const k = key(namespace, name)+ const current = live.get(k)+ // A test that overrides `read` supplies the live object itself, so an+ // object the store never saw is accepted here rather than 404'd.+ if (current && body.metadata.resourceVersion !== current.metadata.resourceVersion) {+ throw { code: 409 }+ }+ const stored = structuredClone(body)+ stored.metadata = {+ ...stored.metadata,+ resourceVersion: String(Number(current?.metadata.resourceVersion ?? '1') + 1),+ generation: 1,+ }+ live.set(k, { ...stored, status: structuredClone(status) })+ return {}+ }+ )+}++const LIVE_CLUSTER_IP = '10.0.0.7'++function installLiveServiceAndConfigMapStores(): void {+ installLiveStore(+ {+ read: mockCoreApi.readNamespacedService,+ create: mockCoreApi.createNamespacedService,+ replace: mockCoreApi.replaceNamespacedService,+ },+ {},+ stored => {+ stored.spec = { ...(stored.spec as Record<string, unknown>), clusterIP: LIVE_CLUSTER_IP }+ }+ )+ installLiveStore(+ {+ read: mockCoreApi.readNamespacedConfigMap,+ create: mockCoreApi.createNamespacedConfigMap,+ replace: mockCoreApi.replaceNamespacedConfigMap,+ },+ {}+ )+}++// Per-object write counts: one reconcile writes several objects of the same+// kind, so a bare call count cannot say which of them was written.+function postsOf(create: ReturnType<typeof vi.fn>, name: string): number {+ return create.mock.calls.filter(+ ([args]) => (args as { body: LiveManifest }).body.metadata.name === name+ ).length+}++function putsOf(replace: ReturnType<typeof vi.fn>, name: string): number {+ return replace.mock.calls.filter(([args]) => (args as { name: string }).name === name).length+}++function readsOf(read: ReturnType<typeof vi.fn>, name: string, namespace: string): number {+ return read.mock.calls.filter(([args]) => {+ const a = args as { name: string; namespace: string }+ return a.name === name && a.namespace === namespace+ }).length+}+describe('WorkflowRecipeReconciler', () => {let reconciler: WorkflowRecipeReconciler@@ -215,12 +333,14 @@ describe('WorkflowRecipeReconciler', () => {vi.clearAllMocks()mockVerifyWorkflowRunProvenance.mockResolvedValue('verified')process.env.CLERUM_NETWORK_POLICY_ENFORCEMENT_CONFIRMED = 'true'- mockAppsApi.readNamespacedDeployment.mockReset()- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1', generation: 1 },- spec: { replicas: 1 },- status: { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 },- })+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedDeployment,+ create: mockAppsApi.createNamespacedDeployment,+ replace: mockAppsApi.replaceNamespacedDeployment,+ },+ { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 }+ )mockAppsApi.readNamespacedStatefulSet.mockReset()mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({metadata: { resourceVersion: '1' },@@ -229,22 +349,31 @@ describe('WorkflowRecipeReconciler', () => {})mockAppsApi.patchNamespacedStatefulSet.mockReset()mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})- mockAppsApi.readNamespacedDaemonSet.mockReset()- mockAppsApi.readNamespacedDaemonSet.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { desiredNumberScheduled: 1, numberReady: 1 },- })- mockBatchApi.readNamespacedCronJob.mockReset()- mockBatchApi.readNamespacedCronJob.mockResolvedValue({- metadata: { resourceVersion: '1' },- spec: { suspend: false },- })- mockBatchApi.readNamespacedJob.mockReset()- mockBatchApi.readNamespacedJob.mockResolvedValue({- metadata: { resourceVersion: '1' },- spec: { completions: 1 },- status: { succeeded: 1 },- })+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedDaemonSet,+ create: mockAppsApi.createNamespacedDaemonSet,+ replace: mockAppsApi.replaceNamespacedDaemonSet,+ },+ { desiredNumberScheduled: 1, numberReady: 1 }+ )+ installLiveStore(+ {+ read: mockBatchApi.readNamespacedCronJob,+ create: mockBatchApi.createNamespacedCronJob,+ replace: mockBatchApi.replaceNamespacedCronJob,+ },+ {}+ )+ installLiveStore(+ {+ read: mockBatchApi.readNamespacedJob,+ create: mockBatchApi.createNamespacedJob,+ replace: mockBatchApi.replaceNamespacedJob,+ },+ { succeeded: 1 }+ )+ installLiveServiceAndConfigMapStores()mockCoreApi.readNamespacedPersistentVolumeClaim.mockReset()mockCoreApi.readNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 404 })mockCustomApi.createNamespacedCustomObject.mockReset()@@ -273,16 +402,16 @@ describe('WorkflowRecipeReconciler', () => {mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(({ name }: { name: string }) => {if (!name.endsWith('-coordinator-to-gfs')) return Promise.reject({ code: 404 })+ // The live policy as WRC last wrote it: owned and converged, so a+ // read-first apply has nothing to write. Tests about an absent policy+ // say so explicitly.+ const converged = buildCoordinatorGfsNetworkPolicy({+ recipeName: name.replace(/-coordinator-to-gfs$/, ''),+ sandboxNamespace: 'sandbox-recipes',+ })return Promise.resolve({- metadata: {- name,- uid: `uid-${name}`,- resourceVersion: '1',- labels: {- 'clerum.io/managed-by': 'wrc',- 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),- },- },+ ...converged,+ metadata: { ...converged.metadata, name, uid: `uid-${name}`, resourceVersion: '1' },})})@@ -500,10 +629,15 @@ describe('WorkflowRecipeReconciler', () => {it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {const recipe = makeRecipe({ status: { phase: 'degraded' } })- mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({- spec: { replicas: 1 },- status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },- })+ mockAppsApi.readNamespacedDeployment+ .mockResolvedValueOnce({+ spec: { replicas: 1 },+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },+ })+ .mockResolvedValueOnce({+ spec: { replicas: 1 },+ status: { readyReplicas: 1, updatedReplicas: 1, availableReplicas: 1 },+ })expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')})@@ -972,9 +1106,9 @@ describe('WorkflowRecipeReconciler', () => {mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!address = '93.184.216.21'if (failure === 'read-disappearance') {- // The workload lane is read-first: start absent so its create can- // race with another creator, whose winner then disappears.- if (lane === 'workload') live.delete(name)+ // Both lanes are read-first: start absent so the create can race+ // with another creator, whose winner then disappears.+ live.delete(name)mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {if (request.body.metadata.name === name) {live.delete(name)@@ -1284,6 +1418,68 @@ describe('WorkflowRecipeReconciler', () => {})})+ // awaiting_policy waits for an operator grant. The grant arrives by event+ // (handleGrantUpdateNotification) or by the 30s credential-refresh floor, so a+ // fixed 5s poll cannot make the state advance. It must keep requeueing, but on+ // the transient backoff path rather than the fixed progress interval.+ describe('SDK-only recipe awaiting operator policy', () => {+ const reconcileAwaitingPolicy = async () => {+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({+ phase: 'awaiting_policy',+ message: 'operator policy pending (policy_not_ready)',+ })+ ;(+ reconciler as unknown as {+ config: { pluginWorkloadSdkEnabled: boolean }+ workflowReconciler: {+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly+ }+ }+ ).config.pluginWorkloadSdkEnabled = true+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly+ }+ }+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }++ const recipe = makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },+ },+ })+ const result = await reconciler.reconcile(recipe)++ // Liveness witness: the awaiting_policy branch ran, and the watcher's+ // projection of this result publishes PluginWorkloadSdkPolicyPending=True.+ expect(reconcilePluginWorkloadSdkOnly).toHaveBeenCalledTimes(1)+ expect(result.pluginWorkloadSdkPolicyPending).toBe(true)+ expect(result.message).toBe('operator policy pending (policy_not_ready)')+ expect(reconciler.projectPluginWorkloadSdk(recipe, result).conditions).toContainEqual(+ expect.objectContaining({+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,+ status: 'True',+ })+ )+ return result+ }++ it('does not requeue at a fixed interval', async () => {+ const result = await reconcileAwaitingPolicy()++ expect(result.requeueFixedInterval).not.toBe(true)+ })++ it('still requeues, on the transient backoff base', async () => {+ const result = await reconcileAwaitingPolicy()++ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ })+ })+it('cleans SDK-only runtime resources after the capability is removed without invoking workflow reconciliation', async () => {const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)const workflowReconcile = vi.fn()@@ -2383,59 +2579,450 @@ describe('WorkflowRecipeReconciler', () => {mockBatchApi.replaceNamespacedJob.mockReset().mockResolvedValue({})})- it('does NOT replace a Deployment whose spec-hash is unchanged', async () => {+ it('replaces a Deployment when the desired spec-hash differs', async () => {const recipe = makeRecipe()const workload = recipe.spec.workloads![0]- // First apply creates the object; capture the manifest WRC writes (carries the hash).- let createdBody: { metadata: { annotations: Record<string, string> }; spec: unknown } | null =- null- mockAppsApi.createNamespacedDeployment.mockImplementation((args: { body: unknown }) => {- createdBody = args.body as typeof createdBody- return Promise.resolve({})+ mockAppsApi.readNamespacedDeployment.mockResolvedValue({+ metadata: { resourceVersion: '9', annotations: { [SPEC_HASH]: 'stale-or-foreign' } },+ spec: {},})+ mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})+await (reconciler as unknown as {ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>}).ensureDeployment(workload, recipe, 'minimal', {})- expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()- // Second apply: object already exists with the SAME hash → must skip the PUT.- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },- spec: createdBody!.spec,+ // A present object with another hash is replaced directly: no POST probe.+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion+ ).toBe('9')+ })++ describe('read-first apply', () => {+ type Manifest = { metadata: { name: string; namespace: string; resourceVersion?: string } }+ const ensureDeployment = (workload: unknown, recipe: unknown) =>+ (+ reconciler as unknown as {+ ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ }+ ).ensureDeployment(workload, recipe, 'minimal', {})+ const ensureStatefulSet = (workload: unknown, recipe: unknown) =>+ (+ reconciler as unknown as {+ ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ }+ ).ensureStatefulSet(workload, recipe, 'minimal', {})++ // Stateful Deployment double: a read of an absent object is a 404, a+ // create of a present one is a 409, and a replace must carry the live+ // resourceVersion — the apiserver's own contract.+ const deploymentStore = () => {+ let live: Manifest | null = null+ mockAppsApi.readNamespacedDeployment.mockImplementation(async () => {+ if (!live) throw { code: 404 }+ return clone(live)+ })+ mockAppsApi.createNamespacedDeployment.mockImplementation(+ async ({ body }: { body: Manifest }) => {+ if (live) throw { code: 409 }+ live = { ...clone(body), metadata: { ...clone(body).metadata, resourceVersion: '1' } }+ return {}+ }+ )+ mockAppsApi.replaceNamespacedDeployment.mockImplementation(+ async ({ body }: { body: Manifest }) => {+ expect(body.metadata.resourceVersion).toBe(live?.metadata.resourceVersion)+ live = clone(body)+ return {}+ }+ )+ return { live: () => live }+ }++ it('reads an absent Deployment first, then sends exactly one POST and no PUT', async () => {+ const recipe = makeRecipe()+ const store = deploymentStore()++ await ensureDeployment(recipe.spec.workloads![0], recipe)++ const created = store.live()!+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledWith({+ name: created.metadata.name,+ namespace: created.metadata.namespace,+ })+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mockAppsApi.readNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(+ mockAppsApi.createNamespacedDeployment.mock.invocationCallOrder[0]+ )+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)})- mockAppsApi.replaceNamespacedDeployment.mockClear()- await (- reconciler as unknown as {- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ it('sends neither POST nor PUT for a live Deployment whose spec-hash is unchanged', async () => {+ const recipe = makeRecipe()+ const store = deploymentStore()+ await ensureDeployment(recipe.spec.workloads![0], recipe)+ const live = store.live()!+ mockAppsApi.readNamespacedDeployment.mockClear()+ mockAppsApi.createNamespacedDeployment.mockClear()+ mockAppsApi.replaceNamespacedDeployment.mockClear()++ await ensureDeployment(recipe.spec.workloads![0], recipe)++ // Witness: the gate read the live object before deciding not to write.+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledWith({+ name: live.metadata.name,+ namespace: live.metadata.namespace,+ })+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)+ })++ // The POST is mocked to 409 only so that the pre-read-first code reaches+ // the gate at all; with the read first, it must never be sent.+ it('propagates an unclassifiable gate-read error and writes nothing', async () => {+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }+ const recipe = makeRecipe({ spec: { workloads: [workload] } })+ const boom = new Error('boom')+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(boom)+ mockCoreApi.createNamespacedService.mockRejectedValue({ code: 409 })++ const outcome = ensureStatefulSet(workload, recipe)++ await expect(outcome).rejects.toBe(boom)+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ })++ // The same failure seen from reconcile(): the phase it lands in is part+ // of the behaviour change, not an implementation detail.+ it('degrades, not fails, a recipe whose gate read hits an unclassifiable error', async () => {+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }+ const recipe = makeRecipe({ spec: { workloads: [workload] } })+ const boom = new Error('boom')+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(boom)++ const result = await reconciler.reconcile(recipe)++ expect(mockCoreApi.readNamespacedService).toHaveBeenCalled()+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)+ // The workload fails like any other non-409 create failure, and the+ // recipe degrades rather than latching at the terminal `failed`:+ // degraded recipes are re-enqueued by the workload status refresh loop.+ expect(result.phase).toBe('degraded')+ expect(result.workloadStatuses).toEqual([+ { id: 'db', phase: 'failed', ready: false, message: 'Error: boom' },+ ])+ })++ it.each([+ ['an apiserver 500', { code: 500 }],+ [+ 'a transport reset',+ Object.assign(new Error('connect ECONNRESET 10.0.0.1:443'), { code: 'ECONNRESET' }),+ ],+ ])('falls through to the replace when the gate read fails with %s', async (_, failure) => {+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }+ const recipe = makeRecipe({ spec: { workloads: [workload] } })+ // The first pass creates the headless Service, so the replace has a+ // live object to read back.+ await ensureStatefulSet(workload, recipe)+ mockCoreApi.createNamespacedService.mockClear()+ mockCoreApi.replaceNamespacedService.mockClear()+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(failure)++ await ensureStatefulSet(workload, recipe)++ // Witness: the PUT went out, so the fall-through ran.+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)+ })+ })++ describe('read-first apply of Services and ConfigMaps', () => {+ type CreateArgs = { namespace: string; body: LiveManifest }++ const clearServiceAndConfigMapCalls = () => {+ for (const fn of [+ mockCoreApi.readNamespacedService,+ mockCoreApi.createNamespacedService,+ mockCoreApi.replaceNamespacedService,+ mockCoreApi.readNamespacedConfigMap,+ mockCoreApi.createNamespacedConfigMap,+ mockCoreApi.replaceNamespacedConfigMap,+ ]) {+ fn.mockClear()}- ).ensureDeployment(workload, recipe, 'minimal', {})+ }+ const onlyPost = (create: ReturnType<typeof vi.fn>): CreateArgs => {+ expect(create).toHaveBeenCalledTimes(1)+ return create.mock.calls[0][0] as CreateArgs+ }++ it('sends neither POST nor PUT for an unchanged recipe-lane Service', async () => {+ const recipe = makeRecipe()+ await reconciler.reconcile(recipe)+ const created = onlyPost(mockCoreApi.createNamespacedService)+ const name = created.body.metadata.name+ clearServiceAndConfigMapCalls()++ await reconciler.reconcile(recipe)++ // Witness: the gate read the live Service before deciding not to write.+ expect(readsOf(mockCoreApi.readNamespacedService, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(0)+ })++ it('keeps the live clusterIP when a changed recipe-lane Service is replaced', async () => {+ await reconciler.reconcile(makeRecipe())+ const created = onlyPost(mockCoreApi.createNamespacedService)+ const name = created.body.metadata.name+ clearServiceAndConfigMapCalls()++ await reconciler.reconcile(+ makeRecipe({+ spec: {+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 9090 },+ ],+ },+ })+ )++ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(1)+ const replaced = mockCoreApi.replaceNamespacedService.mock.calls[0][0] as {+ body: { metadata: { resourceVersion?: string }; spec: { clusterIP?: string } }+ }+ expect(replaced.body.spec.clusterIP).toBe(LIVE_CLUSTER_IP)+ expect(replaced.body.metadata.resourceVersion).toBe('1')+ })++ it('sends neither POST nor PUT for an unchanged workflow-lane Service', async () => {+ const recipe = makeRecipe({+ spec: {+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ steps: [{ id: 'run', run: snippetRun() }],+ },+ })+ const deployWorkflowWorkloads = () =>+ (+ reconciler as unknown as {+ deployWorkflowWorkloads: (r: WorkflowRecipeCRD, n: string) => Promise<unknown>+ }+ ).deployWorkflowWorkloads(recipe, recipe.metadata.name)+ await deployWorkflowWorkloads()+ const created = onlyPost(mockCoreApi.createNamespacedService)+ const name = created.body.metadata.name+ clearServiceAndConfigMapCalls()++ await deployWorkflowWorkloads()- expect(mockAppsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()+ expect(readsOf(mockCoreApi.readNamespacedService, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(0)+ })++ it('sends neither POST nor PUT for an unchanged resource ConfigMap', async () => {+ const res = { id: 'cfg', type: 'configmap' as const, data: { k: 'v' } }+ const recipe = makeRecipe({+ spec: {+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ resources: [res],+ },+ })+ const ensureConfigMap = () =>+ (+ reconciler as unknown as {+ ensureConfigMap: (r: unknown, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ ).ensureConfigMap(res, recipe)+ await ensureConfigMap()+ const created = onlyPost(mockCoreApi.createNamespacedConfigMap)+ const name = created.body.metadata.name+ clearServiceAndConfigMapCalls()++ await ensureConfigMap()++ expect(readsOf(mockCoreApi.readNamespacedConfigMap, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedConfigMap, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedConfigMap, name)).toBe(0)+ })})- it('replaces a Deployment when the desired spec-hash differs', async () => {- const recipe = makeRecipe()- const workload = recipe.spec.workloads![0]+ describe('read-first apply of resource Secrets', () => {+ type SecretManifest = LiveManifest & { data?: Record<string, string> }+ type CreateArgs = { namespace: string; body: SecretManifest }+ type SecretRes = {+ id: string+ type: 'secret'+ data?: Record<string, string>+ generateKeys?: string[]+ }- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '9', annotations: { [SPEC_HASH]: 'stale-or-foreign' } },- spec: {},+ // Local to this describe: many other tests override readNamespacedSecret+ // with their own fixtures, so the stateful store is not installed globally.+ beforeEach(() => {+ installLiveStore(+ {+ read: mockCoreApi.readNamespacedSecret,+ create: mockCoreApi.createNamespacedSecret,+ replace: mockCoreApi.replaceNamespacedSecret,+ },+ {}+ )})- mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})- await (- reconciler as unknown as {- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>+ const recipeWith = (res: SecretRes) =>+ makeRecipe({+ spec: {+ workloads: [+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },+ ],+ resources: [res],+ },+ })+ const ensureSecret = (res: SecretRes, recipe: WorkflowRecipeCRD) =>+ (+ reconciler as unknown as {+ ensureSecret: (r: unknown, recipe: WorkflowRecipeCRD) => Promise<void>+ }+ ).ensureSecret(res, recipe)+ const clearSecretCalls = () => {+ for (const fn of [+ mockCoreApi.readNamespacedSecret,+ mockCoreApi.createNamespacedSecret,+ mockCoreApi.replaceNamespacedSecret,+ ]) {+ fn.mockClear()+ }+ }+ const onlyPost = (): CreateArgs => {+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)+ return mockCoreApi.createNamespacedSecret.mock.calls[0][0] as CreateArgs+ }+ const readLive = (name: string, namespace: string): Promise<SecretManifest> =>+ mockCoreApi.readNamespacedSecret({ name, namespace }) as Promise<SecretManifest>+ // Models an external writer relabelling the live Secret to another recipe.+ const relabelToForeignOwner = async (name: string, namespace: string) => {+ const live = await readLive(name, namespace)+ await mockCoreApi.replaceNamespacedSecret({+ name,+ namespace,+ body: {+ ...live,+ metadata: {+ ...live.metadata,+ labels: {+ ...(live.metadata.labels as Record<string, string>),+ 'clerum.io/recipe': 'other-recipe',+ },+ },+ },+ })+ }++ it('sends neither POST nor PUT for an existing generated-key Secret and keeps its keys', async () => {+ const res: SecretRes = { id: 'creds', type: 'secret', generateKeys: ['password'] }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ const before = await readLive(name, created.namespace)+ clearSecretCalls()++ await ensureSecret(res, recipe)++ // Witness: the gate read the live Secret before deciding not to write.+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)+ const after = await readLive(name, created.namespace)+ expect(after.data).toEqual(before.data)+ })++ it('refuses a generated-key Secret owned by another recipe without writing it', async () => {+ const res: SecretRes = { id: 'creds', type: 'secret', generateKeys: ['password'] }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ await relabelToForeignOwner(name, created.namespace)+ clearSecretCalls()++ await expect(ensureSecret(res, recipe)).rejects.toThrow(+ /not owned by WorkflowRecipe "test-recipe".*clerum\.io\/recipe=other-recipe/+ )++ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)+ })++ it('sends neither POST nor PUT for an unchanged user-data Secret', async () => {+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ clearSecretCalls()++ await ensureSecret(res, recipe)++ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)+ })++ it('replaces a user-data Secret whose data changed, carrying the live resourceVersion', async () => {+ const first: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }+ await ensureSecret(first, recipeWith(first))+ const created = onlyPost()+ const name = created.body.metadata.name+ clearSecretCalls()++ const second: SecretRes = { id: 'api', type: 'secret', data: { token: 'v2' } }+ await ensureSecret(second, recipeWith(second))++ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(1)+ const replaced = mockCoreApi.replaceNamespacedSecret.mock.calls[0][0] as {+ body: SecretManifest & { metadata: { resourceVersion?: string } }}- ).ensureDeployment(workload, recipe, 'minimal', {})+ expect(replaced.body.data).toEqual({ token: Buffer.from('v2').toString('base64') })+ expect(replaced.body.metadata.resourceVersion).toBe('1')+ })- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ it('refuses a user-data Secret owned by another recipe without writing it', async () => {+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }+ const recipe = recipeWith(res)+ await ensureSecret(res, recipe)+ const created = onlyPost()+ const name = created.body.metadata.name+ await relabelToForeignOwner(name, created.namespace)+ clearSecretCalls()++ await expect(ensureSecret(res, recipe)).rejects.toThrow(+ /not owned by WorkflowRecipe "test-recipe".*clerum\.io\/recipe=other-recipe/+ )++ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)+ })})it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {@@ -2850,11 +3437,12 @@ describe('WorkflowRecipeReconciler', () => {).ensureCronJob(workload, recipe, 'minimal', {})expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()- mockBatchApi.createNamespacedCronJob.mockRejectedValue({ code: 409 })- mockBatchApi.readNamespacedCronJob.mockResolvedValue({+ // Second apply: the read-first gate finds the object with the SAME hash.+ mockBatchApi.readNamespacedCronJob.mockClear().mockResolvedValue({metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },spec: createdBody!.spec,})+ mockBatchApi.createNamespacedCronJob.mockClear()mockBatchApi.replaceNamespacedCronJob.mockClear()await (@@ -2863,7 +3451,10 @@ describe('WorkflowRecipeReconciler', () => {}).ensureCronJob(workload, recipe, 'minimal', {})- expect(mockBatchApi.replaceNamespacedCronJob).not.toHaveBeenCalled()+ // Witness: the gate read the live object before deciding not to write.+ expect(mockBatchApi.readNamespacedCronJob).toHaveBeenCalledTimes(1)+ expect(mockBatchApi.createNamespacedCronJob).toHaveBeenCalledTimes(0)+ expect(mockBatchApi.replaceNamespacedCronJob).toHaveBeenCalledTimes(0)})it('does NOT replace a Job whose spec-hash is unchanged', async () => {@@ -2883,11 +3474,12 @@ describe('WorkflowRecipeReconciler', () => {).ensureJob(workload, recipe, 'minimal', {})expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()- mockBatchApi.createNamespacedJob.mockRejectedValue({ code: 409 })- mockBatchApi.readNamespacedJob.mockResolvedValue({+ // Second apply: the read-first gate finds the object with the SAME hash.+ mockBatchApi.readNamespacedJob.mockClear().mockResolvedValue({metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },spec: createdBody!.spec,})+ mockBatchApi.createNamespacedJob.mockClear()mockBatchApi.replaceNamespacedJob.mockClear()await (@@ -2896,7 +3488,10 @@ describe('WorkflowRecipeReconciler', () => {}).ensureJob(workload, recipe, 'minimal', {})- expect(mockBatchApi.replaceNamespacedJob).not.toHaveBeenCalled()+ // Witness: the gate read the live object before deciding not to write.+ expect(mockBatchApi.readNamespacedJob).toHaveBeenCalledTimes(1)+ expect(mockBatchApi.createNamespacedJob).toHaveBeenCalledTimes(0)+ expect(mockBatchApi.replaceNamespacedJob).toHaveBeenCalledTimes(0)})it('does NOT replace a DaemonSet whose spec-hash is unchanged', async () => {@@ -2916,11 +3511,12 @@ describe('WorkflowRecipeReconciler', () => {).ensureDaemonSet(workload, recipe, 'minimal', {})expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()- mockAppsApi.createNamespacedDaemonSet.mockRejectedValue({ code: 409 })- mockAppsApi.readNamespacedDaemonSet.mockResolvedValue({+ // Second apply: the read-first gate finds the object with the SAME hash.+ mockAppsApi.readNamespacedDaemonSet.mockClear().mockResolvedValue({metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },spec: createdBody!.spec,})+ mockAppsApi.createNamespacedDaemonSet.mockClear()mockAppsApi.replaceNamespacedDaemonSet.mockClear()await (@@ -2929,19 +3525,24 @@ describe('WorkflowRecipeReconciler', () => {}).ensureDaemonSet(workload, recipe, 'minimal', {})- expect(mockAppsApi.replaceNamespacedDaemonSet).not.toHaveBeenCalled()+ // Witness: the gate read the live object before deciding not to write.+ expect(mockAppsApi.readNamespacedDaemonSet).toHaveBeenCalledTimes(1)+ expect(mockAppsApi.createNamespacedDaemonSet).toHaveBeenCalledTimes(0)+ expect(mockAppsApi.replaceNamespacedDaemonSet).toHaveBeenCalledTimes(0)})- it('falls open and replaces when the existing object cannot be read (never silently skips)', async () => {+ it('falls open and replaces when a transport failure hides the existing object (never silently skips)', async () => {const recipe = makeRecipe()const workload = recipe.spec.workloads![0]- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })- // First read (idempotency check) fails; the replace path's own read then succeeds.+ // First read (the gate) times out at the socket; the replace path's own read then succeeds.mockAppsApi.readNamespacedDeployment.mockReset()- .mockRejectedValueOnce(new Error('transient read timeout'))+ .mockRejectedValueOnce(+ Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), { code: 'ETIMEDOUT' })+ ).mockResolvedValue({ metadata: { resourceVersion: '9' } })+ mockAppsApi.createNamespacedDeployment.mockClear()mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})await (@@ -2950,7 +3551,12 @@ describe('WorkflowRecipeReconciler', () => {}).ensureDeployment(workload, recipe, 'minimal', {})+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion+ ).toBe('9')})})@@ -3712,10 +4318,22 @@ describe('WorkflowRecipeReconciler', () => {})it('uses create-or-replace pattern on 409 (3.11c)', async () => {- mockAppsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ // Read-first race: the Deployment is absent at the gate read, another writer+ // creates it before our POST (409), and the gate is re-entered on the live object.+ mockAppsApi.readNamespacedDeployment+ .mockReset()+ .mockRejectedValueOnce({ code: 404 })+ .mockResolvedValue({ metadata: { resourceVersion: '7' }, status: { readyReplicas: 1 } })+ mockAppsApi.createNamespacedDeployment.mockReset().mockRejectedValueOnce({ code: 409 })+ mockAppsApi.replaceNamespacedDeployment.mockReset().mockResolvedValue({})+await reconciler.reconcile(makeRecipe())- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalled()- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalled()++ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion+ ).toBe('7')})// ─── Resource Creation ──────────────────────────────────────────@@ -4986,8 +5604,9 @@ describe('WorkflowRecipeReconciler', () => {})it('handles partial failure (some workloads fail) (3.15c)', async () => {+ const storeCreate = mockAppsApi.createNamespacedDeployment.getMockImplementation()!mockAppsApi.createNamespacedDeployment- .mockResolvedValueOnce({})+ .mockImplementationOnce(storeCreate).mockRejectedValueOnce(new Error('quota exceeded'))const recipe = makeRecipe({@@ -5166,10 +5785,14 @@ describe('WorkflowRecipeReconciler', () => {status: { readyReplicas: name === 'db' ? 1 : 0 },}))- mockCoreApi.readNamespacedService.mockResolvedValue({- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },- spec: { clusterIP: '10.0.0.1' },- })+ // The legacy raw Services are live and owned by this recipe.+ for (const legacy of ['db', 'db-headless']) {+ await mockCoreApi.createNamespacedService({+ namespace: 'sandbox-recipes',+ body: { metadata: { name: legacy, labels: { 'clerum.io/recipe': 'existing-db' } } },+ })+ }+ mockCoreApi.createNamespacedService.mockClear()const firstResult = await reconciler.reconcile(recipe)@@ -5199,10 +5822,6 @@ describe('WorkflowRecipeReconciler', () => {spec: { replicas: 1 },status: { readyReplicas: 1 },})- mockCoreApi.readNamespacedService.mockResolvedValue({- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },- spec: { clusterIP: '10.0.0.1' },- })await reconciler.reconcile(recipe)@@ -5282,8 +5901,9 @@ describe('WorkflowRecipeReconciler', () => {// Scoped workload not ready yet → tearing down the raw "api" must be deferred.// The raw Deployment carries this recipe's ownership label so cleanup recognizes- // it as its own (issue #571 F2).- mockAppsApi.readNamespacedDeployment.mockResolvedValue({+ // it as its own (issue #571 F2). The scoped Deployment does not exist until+ // this reconcile creates it, so its read-first gate read is a 404.+ const notReady = {metadata: {resourceVersion: '1',generation: 1,@@ -5291,7 +5911,18 @@ describe('WorkflowRecipeReconciler', () => {},spec: { replicas: 1 },status: { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 0, availableReplicas: 0 },- })+ }+ let scopedCreated = false+ mockAppsApi.readNamespacedDeployment.mockImplementation(async ({ name }: { name: string }) => {+ if (name === scoped && !scopedCreated) throw { code: 404 }+ return structuredClone(notReady)+ })+ mockAppsApi.createNamespacedDeployment.mockImplementation(+ async ({ body }: { body: { metadata: { name: string } } }) => {+ if (body.metadata.name === scoped) scopedCreated = true+ return {}+ }+ )const firstResult = await reconciler.reconcile(recipe)@@ -5388,12 +6019,14 @@ describe('WorkflowRecipeReconciler', () => {await reconciler.reconcile(recipe)- // PVC-mounting Deployment keeps the raw name (adopted), so the scoped Deployment- // is never created and no cleanup/deletion is attempted.- const createdNames = mockAppsApi.createNamespacedDeployment.mock.calls.map(- ([arg]) => arg.body.metadata.name+ // PVC-mounting Deployment keeps the raw name (adopted): the live raw object is+ // replaced in place, the scoped Deployment is never created, and no+ // cleanup/deletion is attempted.+ const replacedNames = mockAppsApi.replaceNamespacedDeployment.mock.calls.map(+ ([arg]) => arg.name)- expect(createdNames).toContain('api')+ expect(replacedNames).toEqual(['api'])+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)expect(mockAppsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()})@@ -7196,6 +7829,59 @@ describe('WorkflowRecipeReconciler', () => {}})+ it('replaces a live ui egress policy whose set changed without a create probe', async () => {+ const policyName = 'ui-egress-test-recipe'+ let address = '93.184.216.10'+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),+ })+ const recipe = uiRecipeWithExternals([{ fqdn: 'api.stripe.com', port: 443 }])+ await rec.reconcile(recipe)+ const created = createdPolicy(policyName)+ expect(created).toBeDefined()+ // Stateful live policy: the first render, then whatever each PUT writes.+ let live: k8s.V1NetworkPolicy = {+ ...structuredClone(created!),+ metadata: { ...created!.metadata, uid: 'uid-ui-egress-test-recipe', resourceVersion: '1' },+ }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) =>+ name === policyName ? Promise.resolve(structuredClone(live)) : Promise.reject({ code: 404 })+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {+ if (name !== policyName) return Promise.resolve({})+ live = {+ ...structuredClone(body),+ metadata: {+ ...body.metadata,+ uid: 'uid-ui-egress-test-recipe',+ resourceVersion: String(Number(live.metadata!.resourceVersion) + 1),+ },+ }+ return Promise.resolve(structuredClone(live))+ }+ )+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ address = '93.184.216.20'++ const result = await rec.reconcile(recipe)++ expect(result.phase).toBe('active')+ // Witness: the caller read the live policy it now hands to the apply.+ expect(+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, policyName, 'sandbox-ui')+ ).toBeGreaterThanOrEqual(1)+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, policyName)).toBe(0)+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, policyName)).toBe(1)+ const cidrs = (live.spec?.egress ?? []).flatMap(rule =>+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)+ )+ expect(cidrs).toContain('93.184.216.20/32')+ })+// H-E (audit): a rename of the external FQDN onto the SAME resolved IP/port// renders identical spec.egress, so the egress-signature gate alone would no-op// and discard the re-attributed state annotation. acc.changed must force the@@ -8142,6 +8828,56 @@ describe('WorkflowRecipeReconciler', () => {expect(result.requeueFixedInterval).toBe(false)})+ // The fixed interval exists for `deploying`, where the controller polls for a+ // Pod it cannot watch and must beat the 240s mcp-host readiness deadline.+ // Plugin Workload SDK policy-pending waits for an operator grant instead, which+ // polling cannot advance, so it must requeue on the backoff path.+ it('requeues a workflow awaiting Plugin Workload SDK policy without a fixed interval', async () => {+ const workflowReconcile = vi.fn().mockResolvedValue({+ phase: 'active',+ message: 'Workflow running',+ workflowPhase: 'running',+ pluginWorkloadSdkPolicyPending: true,+ })+ ;(+ reconciler as unknown as {+ config: { pluginWorkloadSdkEnabled: boolean }+ }+ ).config.pluginWorkloadSdkEnabled = true+ ;(+ reconciler as unknown as {+ workflowReconciler: {+ reconcile: typeof workflowReconcile+ validateWorkflowSpec: () => undefined+ }+ }+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }++ const recipe = makeRecipe({+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ steps: [{ id: 'research', instruction: 'run' }],+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['research'] },+ },+ status: { phase: 'candidate' },+ })++ const result = await reconciler.reconcile(recipe)++ // Liveness witness: the workflow lane ran and carried policy-pending into a+ // result whose projection publishes PluginWorkloadSdkPolicyPending=True.+ expect(workflowReconcile).toHaveBeenCalledTimes(1)+ expect(result.pluginWorkloadSdkPolicyPending).toBe(true)+ expect(reconciler.projectPluginWorkloadSdk(recipe, result).conditions).toContainEqual(+ expect.objectContaining({+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,+ status: 'True',+ })+ )+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)+ expect(result.requeueFixedInterval).not.toBe(true)+ })+it('degrades (retryable) workflow transport workloads when child McpServer pre-deploy fails', async () => {const workflowReconcile = vi.fn().mockResolvedValue({phase: 'deploying',@@ -10613,8 +11349,21 @@ describe('WorkflowRecipeReconciler', () => {})}+ // The apply reads first, so a test about creating the policy must start+ // with it absent; the global default models a converged live policy.+ function coordinatorGfsPolicyAbsent(): void {+ const base = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ (args: { name: string; namespace: string }) =>+ args.name === 'gfs-policy-workflow-coordinator-to-gfs'+ ? Promise.reject({ code: 404 })+ : base(args)+ )+ }+it('creates the exact coordinator GFS policy when publishTargets become present', async () => {stubSteadyWorkflowRuntime()+ coordinatorGfsPolicyAbsent()const recipe = activeWorkflow([{ drive: 'main', target: 'published-results' }])const result = await reconciler.reconcile(recipe)@@ -10903,6 +11652,7 @@ describe('WorkflowRecipeReconciler', () => {},])('converges the present policy before returning from $branch', async testCase => {stubSteadyWorkflowRuntime()+ coordinatorGfsPolicyAbsent()const workflowRuntime = (reconciler as unknown as { workflowReconciler: Record<string, unknown> }).workflowReconciler@@ -11014,6 +11764,125 @@ describe('WorkflowRecipeReconciler', () => {expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})+ describe('read-first apply of the coordinator GFS policy', () => {+ const gfsPolicyName = 'gfs-policy-workflow-coordinator-to-gfs'+ const publishTargets = [{ drive: 'main', target: 'published-results' }]++ // Stateful double for the coordinator GFS policy only; every other+ // NetworkPolicy keeps the behaviour installed by the global beforeEach.+ const installGfsPolicyStore = (seed?: k8s.V1NetworkPolicy) => {+ let live = seed ? structuredClone(seed) : undefined+ const baseRead = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!+ const baseCreate = mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!+ const baseReplace =+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ async (args: { name: string; namespace: string }) => {+ if (args.name !== gfsPolicyName) return baseRead(args)+ if (!live) throw { code: 404 }+ return structuredClone(live)+ }+ )+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(+ async (args: { namespace: string; body: k8s.V1NetworkPolicy }) => {+ if (args.body.metadata?.name !== gfsPolicyName) return baseCreate(args)+ if (live) throw { code: 409 }+ live = {+ ...structuredClone(args.body),+ metadata: { ...args.body.metadata, uid: 'uid-owned-policy', resourceVersion: '1' },+ }+ return structuredClone(live)+ }+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(+ async (args: { name: string; namespace: string; body: k8s.V1NetworkPolicy }) => {+ if (args.name !== gfsPolicyName) return baseReplace(args)+ if (args.body.metadata?.resourceVersion !== live?.metadata?.resourceVersion) {+ throw { code: 409 }+ }+ live = {+ ...structuredClone(args.body),+ metadata: {+ ...args.body.metadata,+ uid: 'uid-owned-policy',+ resourceVersion: String(Number(live!.metadata!.resourceVersion) + 1),+ },+ }+ return structuredClone(live)+ }+ )+ return { live: () => live }+ }+ const clearPolicyCalls = () => {+ for (const fn of [+ mockNetworkingApi.readNamespacedNetworkPolicy,+ mockNetworkingApi.createNamespacedNetworkPolicy,+ mockNetworkingApi.replaceNamespacedNetworkPolicy,+ ]) {+ fn.mockClear()+ }+ }++ it('sends neither POST nor PUT for a converged policy on the next pass', async () => {+ stubSteadyWorkflowRuntime()+ installGfsPolicyStore()+ await reconciler.reconcile(activeWorkflow(publishTargets))+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(1)+ clearPolicyCalls()++ await reconciler.reconcile(activeWorkflow(publishTargets))++ // Witness: the gate read the live policy before deciding not to write.+ expect(+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')+ ).toBe(1)+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ })++ it('replaces a live owned policy whose spec drifted, without a create probe', async () => {+ stubSteadyWorkflowRuntime()+ const drifted = buildCoordinatorGfsNetworkPolicy({+ recipeName: 'gfs-policy-workflow',+ sandboxNamespace: 'sandbox-recipes',+ })+ drifted.metadata = { ...drifted.metadata, uid: 'uid-owned-policy', resourceVersion: '7' }+ drifted.spec!.egress![0].ports = [{ port: 9999, protocol: 'TCP' }]+ const store = installGfsPolicyStore(drifted)+ clearPolicyCalls()++ await reconciler.reconcile(activeWorkflow(publishTargets))++ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(1)+ expect(store.live()?.spec?.egress?.[0].ports).toEqual([{ port: 8087, protocol: 'TCP' }])+ expect(store.live()?.metadata?.resourceVersion).toBe('8')+ })++ it('refuses a homonymous policy owned elsewhere without writing it', async () => {+ stubSteadyWorkflowRuntime()+ installGfsPolicyStore({+ metadata: {+ name: gfsPolicyName,+ uid: 'uid-foreign-policy',+ resourceVersion: '7',+ labels: { 'clerum.io/managed-by': 'operator', 'clerum.io/recipe': 'other-recipe' },+ },+ })+ clearPolicyCalls()++ await expect(reconciler.reconcile(activeWorkflow(publishTargets))).rejects.toThrow(+ 'existing policy is not owned by WRC'+ )++ expect(+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')+ ).toBe(1)+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)+ })+ })+it('refuses to delete a homonymous policy not owned by the exact recipe', async () => {stubSteadyWorkflowRuntime()mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({@@ -11065,6 +11934,7 @@ describe('WorkflowRecipeReconciler', () => {})it('opens coordinator GFS egress once only after a successful first deploy', async () => {+ coordinatorGfsPolicyAbsent()const workflowReconcile = vi.fn().mockResolvedValue({phase: 'deploying',message: 'Workflow infrastructure created',@@ -11691,8 +12561,18 @@ describe('WorkflowRecipeReconciler', () => {// ─── resolveWorkloadResourceName: prefixed names for workflow recipes ──- it('uses the persisted scoped workload instance on 409 update for catalog recipes', async () => {- mockAppsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ it('uses the persisted scoped workload instance on update for catalog recipes', async () => {+ // The scoped Deployment already exists with another spec-hash, so the+ // read-first gate replaces it: both the read and the PUT must target the+ // scoped name, never the bare workload id. Two reads: the gate, then the+ // replace re-reads the live resourceVersion.+ const liveScoped = {+ metadata: { resourceVersion: '4', annotations: { 'clerum.io/spec-hash': 'stale' } },+ spec: {},+ }+ mockAppsApi.readNamespacedDeployment+ .mockResolvedValueOnce(liveScoped)+ .mockResolvedValueOnce(liveScoped)const recipe = makeRecipe({metadata: { name: 'simple-recipe', namespace: 'sandbox-recipes', uid: 'uid-nw' },@@ -11709,6 +12589,11 @@ describe('WorkflowRecipeReconciler', () => {const readCall = mockAppsApi.readNamespacedDeployment.mock.calls[0][0]expect(readCall.name).toBe(expectedName)expect(readCall.name).not.toBe('web-server')+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ const replaced = mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0]+ expect(replaced.name).toBe(expectedName)+ expect(replaced.body.metadata.resourceVersion).toBe('4')})it('patchStatus stores terminal workflowExecution phase for pre-runtime workflow failures', async () => {@@ -12060,11 +12945,14 @@ describe('WorkflowRecipeReconciler', () => {data: { 'signing-secret': 'YmFzZTY0LXNlY3JldA==' },})}- mockAppsApi.readNamespacedDeployment.mockReset()- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: overrides?.deploymentReady === false ? 0 : 1 },- })+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedDeployment,+ create: mockAppsApi.createNamespacedDeployment,+ replace: mockAppsApi.replaceNamespacedDeployment,+ },+ { readyReplicas: overrides?.deploymentReady === false ? 0 : 1 }+ )return recipe}@@ -12102,6 +12990,72 @@ describe('WorkflowRecipeReconciler', () => {expect(allFalse).toBe(true)})+ describe('read-first apply of the gateway objects', () => {+ const ns = 'sandbox-recipes'+ const gateway = 'wf-wh-recipe-webhook-gateway'+ const gatewayConfig = 'wf-wh-recipe-webhook-gateway-config'++ // Two reconciles of the same recipe: the second must find every gateway+ // object live with the desired spec-hash and write none of them.+ const reconcileTwice = async (+ api: {+ read: ReturnType<typeof vi.fn>+ create: ReturnType<typeof vi.fn>+ replace: ReturnType<typeof vi.fn>+ },+ name: string+ ) => {+ const recipe = recipeWithWebhook()+ await reconciler.reconcile(recipe)+ expect(postsOf(api.create, name)).toBe(1)+ api.read.mockClear()+ api.create.mockClear()+ api.replace.mockClear()+ await reconciler.reconcile(recipe)+ }++ it('sends neither POST nor PUT for the unchanged gateway ConfigMap', async () => {+ const api = {+ read: mockCoreApi.readNamespacedConfigMap,+ create: mockCoreApi.createNamespacedConfigMap,+ replace: mockCoreApi.replaceNamespacedConfigMap,+ }+ await reconcileTwice(api, gatewayConfig)++ expect(readsOf(api.read, gatewayConfig, ns)).toBe(1)+ expect(postsOf(api.create, gatewayConfig)).toBe(0)+ expect(putsOf(api.replace, gatewayConfig)).toBe(0)+ })++ it('sends neither POST nor PUT for the unchanged gateway Deployment', async () => {+ const api = {+ read: mockAppsApi.readNamespacedDeployment,+ create: mockAppsApi.createNamespacedDeployment,+ replace: mockAppsApi.replaceNamespacedDeployment,+ }+ await reconcileTwice(api, gateway)++ // The readiness check reads the gateway Deployment too; the gate read+ // is the one that precedes it.+ expect(readsOf(api.read, gateway, ns)).toBe(2)+ expect(postsOf(api.create, gateway)).toBe(0)+ expect(putsOf(api.replace, gateway)).toBe(0)+ })++ it('sends neither POST nor PUT for the unchanged gateway Service', async () => {+ const api = {+ read: mockCoreApi.readNamespacedService,+ create: mockCoreApi.createNamespacedService,+ replace: mockCoreApi.replaceNamespacedService,+ }+ await reconcileTwice(api, gateway)++ expect(readsOf(api.read, gateway, ns)).toBe(1)+ expect(postsOf(api.create, gateway)).toBe(0)+ expect(putsOf(api.replace, gateway)).toBe(0)+ })+ })+it('T6 (#575) converges the 3 gateway NetworkPolicies through live reads', async () => {mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })@@ -12336,11 +13290,14 @@ describe('WorkflowRecipeReconciler', () => {data: { 'signing-secret': 'YmFzZTY0LXNlY3JldA==' },})}- mockAppsApi.readNamespacedDeployment.mockReset()- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 1 },- })+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedDeployment,+ create: mockAppsApi.createNamespacedDeployment,+ replace: mockAppsApi.replaceNamespacedDeployment,+ },+ { readyReplicas: 1 }+ )return recipe}@@ -13161,10 +14118,14 @@ describe('WorkflowRecipeReconciler', () => {})it('re-reconciles a recipe failed by shared mcp-server internal dependency boundary', async () => {- mockAppsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 1 },- })+ installLiveStore(+ {+ read: mockAppsApi.readNamespacedDeployment,+ create: mockAppsApi.createNamespacedDeployment,+ replace: mockAppsApi.replaceNamespacedDeployment,+ },+ { readyReplicas: 1 }+ )const recipe = makeRecipe({spec: {workloads: [diff --git a/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts b/workflow-recipes/src/reconciler/uiEgressMetadata.test.tsindex 5580e5ca9..bbd7f74a2 100644--- a/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts+++ b/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts@@ -108,7 +108,8 @@ describe('UI egress dedicated writer metadata and lifecycle', () => {await fixture.reconcile()expect(fixture.read).toHaveBeenCalledTimes(2)- expect(fixture.create).toHaveBeenCalledTimes(1)+ // The policy already exists, so the read-first apply replaces it without a create probe.+ expect(fixture.create).not.toHaveBeenCalled()expect(fixture.replace).toHaveBeenCalledTimes(1)expect(fixture.live().metadata).toMatchObject({labels: {@@ -173,7 +174,7 @@ describe('UI egress dedicated writer metadata and lifecycle', () => {),})expect(fixture.read).toHaveBeenCalledTimes(2)- expect(fixture.create).toHaveBeenCalledTimes(1)+ expect(fixture.create).not.toHaveBeenCalled()expect(fixture.replace).not.toHaveBeenCalled()})diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex f4e9afc6a..4b3c90294 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1104,7 +1104,8 @@ export class WorkflowRecipeReconciler {const result = await waitForExternalEgressReady(this.delegationDeps,externalServers,- namespace+ namespace,+ recipe.metadata.name)if (!result.ready) {throw externalEgressReadinessError(recipe.metadata.name, result)@@ -1116,7 +1117,8 @@ export class WorkflowRecipeReconciler {const { ready, pending } = await waitForNetworkReady(this.delegationDeps,genericServers,- namespace+ namespace,+ recipe.metadata.name)if (!ready) {createLogger('wrc', recipe.metadata.name).warn('Generic network readiness not confirmed', {@@ -2260,14 +2262,17 @@ export class WorkflowRecipeReconciler {// degrade the poll cadence toward the 60s cap and re-expose the 240s// mcp-host readiness deadline this requeue exists to beat (see §5 and// requeueFixedInterval doc).+ //+ // Plugin Workload SDK policy-pending also requeues, but on the backoff+ // path: it waits for an operator grant, which arrives by event or by the+ // 30s credential-refresh floor. Polling cannot advance it, and it has no+ // deadline that bounds a fixed-interval loop.requeueAfterMs: result.skipStatusPatch? TRANSIENT_REQUEUE_BASE_MS: result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending? WORKFLOW_PROGRESS_REQUEUE_BASE_MS: undefined,- requeueFixedInterval:- !result.skipStatusPatch &&- (result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending === true),+ requeueFixedInterval: !result.skipStatusPatch && result.phase === 'deploying',internalDependencyConditions: workflowInternalDependencyConditions,secretOwnershipConditions: workflowSecretOwnershipConditions,}@@ -2736,7 +2741,12 @@ export class WorkflowRecipeReconciler {body: svc,})},- `Service "${svcName}"`+ `Service "${svcName}"`,+ {+ manifest: svc,+ readExisting: () =>+ this.coreApi.readNamespacedService({ name: svcName, namespace: svcNs }),+ })}}@@ -2938,8 +2948,9 @@ export class WorkflowRecipeReconciler {sdkOnlyBootstrapPending? TRANSIENT_REQUEUE_BASE_MS: undefined,- requeueFixedInterval:- sdkOnlyPolicyPending || sdkOnlyRuntime?.phase === 'deploying' || sdkOnlyBootstrapPending,+ // Policy-pending waits for an operator grant (event or the 30s refresh+ // floor), so it requeues with backoff; only boot progress is fixed-interval.+ requeueFixedInterval: sdkOnlyRuntime?.phase === 'deploying' || sdkOnlyBootstrapPending,}} catch (error) {// Transient failures must not brick a healthy recipe at the terminal,@@ -3286,7 +3297,15 @@ export class WorkflowRecipeReconciler {body: built.configConfigMap,})},- `WebhookGateway ConfigMap "${gatewayConfigMapName(recipeName)}"`+ `WebhookGateway ConfigMap "${gatewayConfigMapName(recipeName)}"`,+ {+ manifest: built.configConfigMap,+ readExisting: () =>+ this.coreApi.readNamespacedConfigMap({+ name: gatewayConfigMapName(recipeName),+ namespace: ns,+ }),+ })await this.createOrReplace(@@ -3303,7 +3322,15 @@ export class WorkflowRecipeReconciler {body: built.deployment,})},- `WebhookGateway Deployment "${gatewayResourceName(recipeName)}"`+ `WebhookGateway Deployment "${gatewayResourceName(recipeName)}"`,+ {+ manifest: built.deployment,+ readExisting: () =>+ this.appsApi.readNamespacedDeployment({+ name: gatewayResourceName(recipeName),+ namespace: ns,+ }),+ })await this.createOrReplace(@@ -3321,7 +3348,15 @@ export class WorkflowRecipeReconciler {body: built.service,})},- `WebhookGateway Service "${gatewayServiceName(recipeName)}"`+ `WebhookGateway Service "${gatewayServiceName(recipeName)}"`,+ {+ manifest: built.service,+ readExisting: () =>+ this.coreApi.readNamespacedService({+ name: gatewayServiceName(recipeName),+ namespace: ns,+ }),+ })// The gateway policies use the same live convergence path as other@@ -3596,7 +3631,12 @@ export class WorkflowRecipeReconciler {body: svc,})},- `Service "${svcName}"`+ `Service "${svcName}"`,+ {+ manifest: svc,+ readExisting: () =>+ this.coreApi.readNamespacedService({ name: svcName, namespace: svcNs }),+ })} catch (err) {createLogger('wrc', recipe.metadata.name).error(@@ -4023,15 +4063,18 @@ export class WorkflowRecipeReconciler {/*** Create-or-replace a managed object. When `idempotency` is supplied, the- * desired manifest is stamped with a spec-hash annotation and, on the- * already-exists (409) path, the existing object's stamped hash is compared to- * the desired hash — if they match the replace PUT is SKIPPED entirely.+ * desired manifest is stamped with a spec-hash annotation and the object is+ * READ FIRST: absent → POST, present with the desired hash → no write at all,+ * present with another hash → PUT. Without `idempotency` the object is+ * created first and replaced on 409.** This is what stops the generation churn: WRC reconciles every workload on a* periodic resync, and an unconditional full replace re-defaults server-managed* fields, bumping metadata.generation with no real change (→ a degraded↔active* status flap and a downstream HCC NetworkPolicy no-op write storm). Skipping- * unchanged writes makes the reconcile idempotent. See specHash.ts.+ * unchanged writes makes the reconcile idempotent, and reading first keeps the+ * steady state from sending a POST only to have it rejected with 409.+ * See specHash.ts.*/private async createOrReplace(createFn: () => Promise<unknown>,@@ -4042,54 +4085,81 @@ export class WorkflowRecipeReconciler {readExisting: () => Promise<{ metadata?: { annotations?: { [key: string]: string } } } | null>}): Promise<void> {- if (idempotency) stampSpecHash(idempotency.manifest)+ if (idempotency) {+ stampSpecHash(idempotency.manifest)+ const gate = await this.applyGate(idempotency)+ if (gate === 'unchanged') {+ createLogger('wrc', 'workflow-recipes').info(+ 'Resource spec hash unchanged; skipping update',+ { label }+ )+ return+ }+ if (gate === 'changed') {+ await this.replaceResource(replaceFn, label)+ return+ }+ }try {await createFn()createLogger('wrc', 'workflow-recipes').info('Created resource', { label })} catch (error: unknown) {- if (getErrorCode(error) === 409) {- if (idempotency && (await this.applyIsNoop(idempotency))) {- createLogger('wrc', 'workflow-recipes').info(- 'Resource spec hash unchanged; skipping update',- { label }- )- return- }- try {- await replaceFn()- createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })- } catch (updateError) {- createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {- label,- err: updateError,- })- throw updateError- }- } else {+ if (getErrorCode(error) !== 409) {createLogger('wrc', 'workflow-recipes').error('Failed to create resource', {label,err: error,})throw error}+ // Without a gate this is the create-first probe. With one, another writer+ // created the object between our read and this POST: re-enter the gate.+ if (idempotency && (await this.applyGate(idempotency)) === 'unchanged') {+ createLogger('wrc', 'workflow-recipes').info(+ 'Resource spec hash unchanged; skipping update',+ { label }+ )+ return+ }+ await this.replaceResource(replaceFn, label)+ }+ }++ private async replaceResource(replaceFn: () => Promise<unknown>, label: string): Promise<void> {+ try {+ await replaceFn()+ createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })+ } catch (updateError) {+ createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {+ label,+ err: updateError,+ })+ throw updateError}}/**- * True when the existing object already carries the desired spec-hash, so the- * replace would be a no-op. A read failure returns false (fall through to- * replace) — never skip an update we cannot prove is unnecessary.+ * Read the live object and decide the write: `absent` (404) → create,+ * `unchanged` (it carries the desired spec-hash) → no write, `changed` → replace.+ *+ * A read failure that carries an apiserver status or a transport signature+ * returns `changed` — never skip an update we cannot prove is unnecessary.+ * Any other failure is not a read outcome but a domain or programming error,+ * and it propagates instead of turning into a silent overwrite.*/- private async applyIsNoop(idempotency: {+ private async applyGate(idempotency: {manifest: { metadata?: { annotations?: { [key: string]: string } } }readExisting: () => Promise<{ metadata?: { annotations?: { [key: string]: string } } } | null>- }): Promise<boolean> {+ }): Promise<'absent' | 'unchanged' | 'changed'> {+ let existing: { metadata?: { annotations?: { [key: string]: string } } } | nulltry {- const existing = await idempotency.readExisting()- return specHashUnchanged(idempotency.manifest, existing ?? null)- } catch {- return false+ existing = await idempotency.readExisting()+ } catch (error) {+ if (getErrorCode(error) === 404) return 'absent'+ if (getErrorCode(error) !== undefined || isRetryableInfraError(error)) return 'changed'+ throw error}+ if (existing == null) return 'absent'+ return specHashUnchanged(idempotency.manifest, existing) ? 'unchanged' : 'changed'}private stableComparableString(value: unknown): string {@@ -4617,7 +4687,7 @@ export class WorkflowRecipeReconciler {ns,})} else {- await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name, existing)// #299: the policy has landed — record the set it actually enforces.this.logResolvedEgressSet(recipe.metadata.name,@@ -5416,51 +5486,65 @@ export class WorkflowRecipeReconciler {return live}+ /**+ * `existing` is the live policy the caller already read this pass (or null+ * when there is none). When one exists the create probe is skipped and the+ * replace runs directly; the replace still re-reads and fences on that fresh+ * resourceVersion, because a contraction earlier in the same pass may have+ * written the policy after the caller's snapshot.+ */private async applyOwnedRecipeNetworkPolicy(policy: k8s.V1NetworkPolicy,namespace: string,- recipeName: string+ recipeName: string,+ existing: k8s.V1NetworkPolicy | null): Promise<void> {const policyName = policy.metadata?.nameif (!policyName) throw new Error('Recipe NetworkPolicy requires a name')- await this.createOrReplace(- () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),- async () => {- let live: k8s.V1NetworkPolicy- try {- live = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace,- })- } catch (error: unknown) {- if (getErrorCode(error) === 404) {- throw new NetworkPolicyReplanRequiredError(- `NetworkPolicy "${policyName}" in ${namespace} disappeared after create conflict; a fresh reconciliation is required`,- { cause: error }- )- }- throw error- }- this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)- if (!live.metadata?.resourceVersion) {- throw new NetworkPolicyOwnershipConflictError(- `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`+ const label = `NetworkPolicy "${policyName}" in ${namespace}`+ const replaceLive = async () => {+ let live: k8s.V1NetworkPolicy+ try {+ live = await this.networkingApi.readNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ } catch (error: unknown) {+ if (getErrorCode(error) === 404) {+ throw new NetworkPolicyReplanRequiredError(+ `NetworkPolicy "${policyName}" in ${namespace} disappeared before replace; a fresh reconciliation is required`,+ { cause: error })}- const replacement = buildNetworkPolicyReplacement(policy, live)- try {- return await this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace,- body: replacement,- })- } catch (error: unknown) {- // This PUT is fenced by the freshly checked owner/resourceVersion.- // Losing that snapshot is unfinished convergence, not bad user intent.- throwNetworkPolicyMutationError(error, policyName, namespace)- }- },- `NetworkPolicy "${policyName}" in ${namespace}`+ throw error+ }+ this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)+ if (!live.metadata?.resourceVersion) {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`+ )+ }+ const replacement = buildNetworkPolicyReplacement(policy, live)+ try {+ return await this.networkingApi.replaceNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ body: replacement,+ })+ } catch (error: unknown) {+ // This PUT is fenced by the freshly checked owner/resourceVersion.+ // Losing that snapshot is unfinished convergence, not bad user intent.+ throwNetworkPolicyMutationError(error, policyName, namespace)+ }+ }+ if (existing) {+ await this.replaceResource(replaceLive, label)+ return+ }+ await this.createOrReplace(+ () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),+ replaceLive,+ label)}@@ -5879,30 +5963,60 @@ export class WorkflowRecipeReconciler {const namespace = this.config.sandboxNamespaceconst policy = this.coordinatorGfsNetworkPolicy(recipe)const name = policy.metadata!.name!+ const label = `NetworkPolicy "${name}" in ${namespace}`+ const matchesDesired = (existing: k8s.V1NetworkPolicy): boolean => {+ const desiredLabels = policy.metadata?.labels ?? {}+ const existingLabels = existing.metadata?.labels ?? {}+ const labelsMatch = Object.entries(desiredLabels).every(+ ([key, value]) => existingLabels[key] === value+ )+ return labelsMatch && JSON.stringify(existing.spec) === JSON.stringify(policy.spec)+ }+ const replaceWith = (existing: k8s.V1NetworkPolicy) => {+ policy.metadata!.resourceVersion = existing.metadata?.resourceVersion+ return this.networkingApi.replaceNamespacedNetworkPolicy({+ name,+ namespace,+ body: policy,+ })+ }++ // Read first so a converged policy costs one GET instead of a POST that+ // the apiserver rejects with 409. Only a 404 falls through to the create+ // path; any other read failure propagates.+ let existing: k8s.V1NetworkPolicy | null+ try {+ existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })+ } catch (error: unknown) {+ if (getErrorCode(error) !== 404) throw error+ existing = null+ }+ if (existing) {+ this.assertCoordinatorGfsNetworkPolicyOwnership(existing, recipe.metadata.name, namespace)+ if (matchesDesired(existing)) {+ createLogger('wrc', recipe.metadata.name).info(+ 'NetworkPolicy matches desired state; skipping update',+ { label }+ )+ return+ }+ const live = existing+ await this.replaceResource(() => replaceWith(live), label)+ return+ }+await this.createOrReplace(() => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),async () => {- const existing = await this.networkingApi.readNamespacedNetworkPolicy({- name,- namespace,- })- this.assertCoordinatorGfsNetworkPolicyOwnership(existing, recipe.metadata.name, namespace)- const desiredLabels = policy.metadata?.labels ?? {}- const existingLabels = existing.metadata?.labels ?? {}- const labelsMatch = Object.entries(desiredLabels).every(- ([key, value]) => existingLabels[key] === value- )- if (labelsMatch && JSON.stringify(existing.spec) === JSON.stringify(policy.spec)) {- return existing- }- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion- return this.networkingApi.replaceNamespacedNetworkPolicy({+ const raced = await this.networkingApi.readNamespacedNetworkPolicy({name,namespace,- body: policy,})+ this.assertCoordinatorGfsNetworkPolicyOwnership(raced, recipe.metadata.name, namespace)+ if (matchesDesired(raced)) return raced+ return replaceWith(raced)},- `NetworkPolicy "${name}" in ${namespace}`+ label)}@@ -7019,7 +7133,46 @@ export class WorkflowRecipeReconciler {ns,recipe.metadata.namespace ?? this.config.sandboxNamespace)- if (res.generateKeys && res.generateKeys.length > 0) {+ const label = `Secret "${res.id}" in ${ns}`+ const generatesKeys = Boolean(res.generateKeys && res.generateKeys.length > 0)++ // Read first, compared structurally rather than by spec-hash: a hash of+ // user data written to an annotation would expose it to offline guessing,+ // and generated keys differ on every build so a hash would never match.+ // Only a 404 falls through to the create path below; any other read+ // failure propagates.+ let existing: k8s.V1Secret | null+ try {+ existing = await this.coreApi.readNamespacedSecret({ name, namespace: ns })+ } catch (error) {+ if (getErrorCode(error) !== 404) throw error+ existing = null+ }+ if (existing) {+ this.assertExistingResourceOwnedByRecipe(existing, name, res.id, recipe, ns, 'Secret')+ if (generatesKeys) {+ createLogger('wrc', 'workflow-recipes').info(+ 'Secret with generated keys exists; keeping its keys',+ { label }+ )+ return+ }+ if (this.secretMatchesDesired(manifest, existing)) {+ createLogger('wrc', 'workflow-recipes').info(+ 'Secret matches desired state; skipping update',+ { label }+ )+ return+ }+ manifest.metadata!.resourceVersion = existing.metadata?.resourceVersion+ await this.replaceResource(+ () => this.coreApi.replaceNamespacedSecret({ name, namespace: ns, body: manifest }),+ label+ )+ return+ }++ if (generatesKeys) {// Create-only semantics: random credentials are generated once and must not be// rotated on subsequent reconcile loops (rotation would silently break running// workloads that have the original values mounted via envFrom/envSecret).@@ -7032,7 +7185,7 @@ export class WorkflowRecipeReconciler {const existing = await this.coreApi.readNamespacedSecret({ name, namespace: ns })this.assertExistingResourceOwnedByRecipe(existing, name, res.id, recipe, ns, 'Secret')},- `Secret "${res.id}" in ${ns}`+ label)} else {await this.createOrReplace(@@ -7049,11 +7202,34 @@ export class WorkflowRecipeReconciler {body: manifest,})},- `Secret "${res.id}" in ${ns}`+ label)}}+ /**+ * A live Secret needs no write when its data and type equal the desired ones,+ * it carries every desired label, and its controller ownerReference points at+ * the same owner uid (none on either side for a cross-namespace Secret).+ */+ private secretMatchesDesired(desired: k8s.V1Secret, existing: k8s.V1Secret): boolean {+ if (+ this.stableComparableString(desired.data ?? {}) !==+ this.stableComparableString(existing.data ?? {})+ ) {+ return false+ }+ if (desired.type !== existing.type) return false+ const liveLabels = existing.metadata?.labels ?? {}+ const labelsMatch = Object.entries(desired.metadata?.labels ?? {}).every(+ ([key, value]) => liveLabels[key] === value+ )+ if (!labelsMatch) return false+ const controllerUid = (secret: k8s.V1Secret) =>+ secret.metadata?.ownerReferences?.find(ref => ref.controller)?.uid+ return controllerUid(desired) === controllerUid(existing)+ }+/*** Refuse to mutate a pre-existing Secret/ConfigMap that is not owned by this* recipe (`clerum.io/recipe` label). Mirrors assertExistingResourcePvcOwnedByRecipe@@ -7199,7 +7375,15 @@ export class WorkflowRecipeReconciler {body: manifest,})},- `ConfigMap "${res.id}" in ${ns}`+ `ConfigMap "${res.id}" in ${ns}`,+ // An unchanged hash skips the ownership assert along with the write. Only+ // an object carrying this manifest's hash can match, and that hash covers+ // the `clerum.io/recipe` label the assert reads; forging the annotation+ // takes the same access as forging that label.+ {+ manifest,+ readExisting: () => this.coreApi.readNamespacedConfigMap({ name, namespace: ns }),+ })}diff --git a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.tsindex 89e010ef0..d249886d2 100644--- a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts+++ b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts@@ -52,7 +52,7 @@ const mockCoreApi = {}),deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),createNamespacedSecret: vi.fn().mockResolvedValue({}),- readNamespacedSecret: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),+ readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),replaceNamespacedSecret: vi.fn().mockResolvedValue({}),deleteNamespacedSecret: vi.fn().mockResolvedValue({}),createNamespacedConfigMap: vi.fn().mockResolvedValue({}),@@ -105,6 +105,12 @@ describe('Workflow Reconciler Bifurcation', () => {beforeEach(() => {vi.clearAllMocks()+ // Secrets are applied read-first, so a test that fails before its POST+ // leaves its mock*Once queue unconsumed, and clearAllMocks keeps that queue+ // for the next test. Reset the Secret doubles so no premise crosses tests.+ // By default no Secret exists, the state of a freshly approved recipe.+ mockCoreApi.createNamespacedSecret.mockReset().mockResolvedValue({})+ mockCoreApi.readNamespacedSecret.mockReset().mockRejectedValue({ code: 404 })mockAppsApi.readNamespacedDeployment.mockResolvedValue({metadata: { resourceVersion: '1', generation: 1 },spec: { replicas: 1 },@@ -156,6 +162,8 @@ describe('Workflow Reconciler Bifurcation', () => {})it('continues with workload path when spec.steps is absent', async () => {+ // The Deployment does not exist yet: the read-first gate read is a 404.+ mockAppsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })const recipe: WorkflowRecipeCRD = {apiVersion: 'clerum.io/v1alpha1',kind: 'WorkflowRecipe',@@ -172,6 +180,8 @@ describe('Workflow Reconciler Bifurcation', () => {})it('accepts a pre-existing recipe PVC only when its WRC ownership labels match', async () => {+ // The Deployment does not exist yet: the read-first gate read is a 404.+ mockAppsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })mockCoreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code: 409 })mockCoreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({metadata: {@@ -321,8 +331,10 @@ describe('Workflow Reconciler Bifurcation', () => {})it('refuses to overwrite a Secret owned by another recipe (issue #571 F1b)', async () => {- mockCoreApi.createNamespacedSecret.mockRejectedValueOnce({ code: 409 })- mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({+ // Every Secret name reads back as another recipe's Secret: the raw-name+ // adoption probe declines it, and the read-first apply of the scoped name+ // must refuse it on that read, before any write.+ mockCoreApi.readNamespacedSecret.mockResolvedValue({metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },})const recipe: WorkflowRecipeCRD = {@@ -340,14 +352,31 @@ describe('Workflow Reconciler Bifurcation', () => {expect(result.phase).toBe('failed')expect(result.message).toContain('not owned by WorkflowRecipe "normal-recipe"')+ // Liveness witness: the scoped Secret was read, so the refusal came from+ // the read-first apply and not from an earlier failure.+ expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({+ name: expect.stringMatching(/^normal-recipe-creds-[a-f0-9]{12}$/),+ namespace: 'sandbox-recipes',+ })+ expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled()expect(mockCoreApi.replaceNamespacedSecret).not.toHaveBeenCalled()})it('refuses to inherit a generateKeys Secret owned by another recipe on 409 (issue #571 S3)', async () => {- mockCoreApi.createNamespacedSecret.mockRejectedValueOnce({ code: 409 })- mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },+ // Race: the read-first apply finds no scoped Secret, another recipe creates+ // it before our POST, and the POST answers 409. The re-read then returns+ // the foreign Secret.+ const scopedName = /^normal-recipe-gen-creds-[a-f0-9]{12}$/+ let scopedReads = 0+ mockCoreApi.readNamespacedSecret.mockImplementation(({ name }: { name: string }) => {+ if (!scopedName.test(name)) return Promise.reject({ code: 404 })+ scopedReads += 1+ if (scopedReads === 1) return Promise.reject({ code: 404 })+ return Promise.resolve({+ metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },+ })})+ mockCoreApi.createNamespacedSecret.mockRejectedValue({ code: 409 })const recipe: WorkflowRecipeCRD = {apiVersion: 'clerum.io/v1alpha1',kind: 'WorkflowRecipe',@@ -365,6 +394,9 @@ describe('Workflow Reconciler Bifurcation', () => {// silently inheriting a foreign Secret.expect(result.phase).toBe('failed')expect(result.message).toContain('not owned by WorkflowRecipe "normal-recipe"')+ // Liveness witness: the POST ran and the 409 was followed by a re-read.+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)+ expect(scopedReads).toBe(2)})it('rejects prepareVolumeOwnership without a non-root target UID', async () => {