Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(workflow-recipes): back off policy-pending requeues and read before apply

Alfredo Lopeze713760866dd
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 2104b456b..f2d9bb5a5 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.364",
+ "version": "0.1.365",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.364",
+ "version": "0.1.365",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 334b84f31..df23a6dda 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.364",
+ "version": "0.1.365",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
index a49b9a9c7..2ae4f152d 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts
@@ -5,6 +5,7 @@
*/
import { describe, expect, it, vi } from 'vitest'
import { WorkflowRecipeCRD } from '../types'
+import { captureLogger } from './__tests__/captureLogger'
import {
DelegationDeps,
PRE_DEPLOY_ANNOTATION,
@@ -1885,6 +1886,7 @@ describe('delegateTransportWorkloads', () => {
.mockRejectedValueOnce({ code: 404 }),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -1942,7 +1944,10 @@ describe('delegateTransportWorkloads', () => {
),
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }
+ const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ createNamespacedService: vi.fn().mockResolvedValue({}),
+ }
const deps: DelegationDeps = {
customApi: mockCustomApi as unknown as DelegationDeps['customApi'],
coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],
@@ -2017,7 +2022,10 @@ describe('delegateTransportWorkloads', () => {
),
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }
+ const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ createNamespacedService: vi.fn().mockResolvedValue({}),
+ }
const deps: DelegationDeps = {
customApi: mockCustomApi as unknown as DelegationDeps['customApi'],
coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],
@@ -2063,7 +2071,10 @@ describe('delegateTransportWorkloads', () => {
}),
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }
+ const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ createNamespacedService: vi.fn().mockResolvedValue({}),
+ }
const deps: DelegationDeps = {
customApi: mockCustomApi as unknown as DelegationDeps['customApi'],
coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],
@@ -2113,7 +2124,10 @@ describe('delegateTransportWorkloads', () => {
}),
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
- const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }
+ const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
+ createNamespacedService: vi.fn().mockResolvedValue({}),
+ }
const deps: DelegationDeps = {
customApi: mockCustomApi as unknown as DelegationDeps['customApi'],
coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],
@@ -2157,6 +2171,7 @@ describe('delegateTransportWorkloads', () => {
patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2175,6 +2190,7 @@ describe('delegateTransportWorkloads', () => {
createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2208,6 +2224,7 @@ describe('delegateTransportWorkloads', () => {
.mockRejectedValueOnce({ code: 404 }),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2236,6 +2253,7 @@ describe('delegateTransportWorkloads', () => {
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2263,7 +2281,7 @@ describe('delegateTransportWorkloads', () => {
})
it('rejects an existing McpServer owned by a different recipe label', async () => {
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const errorLog = captureLogger('error')
const mockCustomApi = {
createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
getNamespacedCustomObject: vi.fn().mockResolvedValueOnce({
@@ -2272,6 +2290,7 @@ describe('delegateTransportWorkloads', () => {
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2283,12 +2302,20 @@ describe('delegateTransportWorkloads', () => {
await expect(
delegateTransportWorkloads(deps, makeRecipe(), 'mcp-server', new Map())
).rejects.toThrow(/Delegation failed for workload\(s\): redis-mcp/)
+ // Witness: the aggregate rejection came from the ownership guard, not from
+ // any other failure on the way to it.
+ expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {
+ workloadId: 'redis-mcp',
+ error: expect.objectContaining({
+ message: expect.stringContaining('already exists for recipe "other-recipe"'),
+ }),
+ })
expect(mockCustomApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalledWith(
expect.objectContaining({ plural: 'contexts' })
)
} finally {
- errorSpy.mockRestore()
+ errorLog.mockRestore()
}
})
@@ -2313,6 +2340,7 @@ describe('delegateTransportWorkloads', () => {
.mockResolvedValueOnce({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2336,7 +2364,7 @@ describe('delegateTransportWorkloads', () => {
})
it('keeps delegation non-fatal when a recipe-owned McpServer remains after conflict retries', async () => {
- const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const warnLog = captureLogger('warn')
const recipe = makeRecipe()
const desiredManifest = buildMcpServerManifest(recipe.spec.workloads![0], recipe, 'mcp-server')!
const mockCustomApi = {
@@ -2363,6 +2391,7 @@ describe('delegateTransportWorkloads', () => {
.mockRejectedValueOnce({ code: 409 }),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2375,17 +2404,20 @@ describe('delegateTransportWorkloads', () => {
expect(delegated).toEqual(['test-recipe-redis-mcp'])
expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)
- expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('existing spec already matches'))
+ expect(warnLog).toHaveBeenCalledWith(
+ expect.stringContaining('existing spec already matches'),
+ { mcpServer: 'test-recipe-redis-mcp', retries: 3 }
+ )
expect(mockCustomApi.createNamespacedCustomObject).toHaveBeenCalledWith(
expect.objectContaining({ plural: 'contexts' })
)
} finally {
- warnSpy.mockRestore()
+ warnLog.mockRestore()
}
})
it('does not hide exhausted McpServer conflicts when the remaining recipe-owned object is stale', async () => {
- const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+ const errorLog = captureLogger('error')
const recipe = makeRecipe()
const desiredManifest = buildMcpServerManifest(recipe.spec.workloads![0], recipe, 'mcp-server')!
const staleSpec = { ...(desiredManifest.spec as Record<string, unknown>), image: 'old:image' }
@@ -2413,6 +2445,7 @@ describe('delegateTransportWorkloads', () => {
.mockRejectedValueOnce({ code: 409 }),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2424,11 +2457,20 @@ describe('delegateTransportWorkloads', () => {
await expect(
delegateTransportWorkloads(deps, recipe, 'mcp-server', new Map())
).rejects.toThrow(/Delegation failed for workload\(s\): redis-mcp/)
+ // Witness: the rejection is the exhausted-conflict error, after all three
+ // replace attempts, not an earlier failure.
+ expect(mockCustomApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)
+ expect(errorLog).toHaveBeenCalledWith('Failed to delegate workload', {
+ workloadId: 'redis-mcp',
+ error: expect.objectContaining({
+ message: expect.stringContaining('failed to update after conflict retries'),
+ }),
+ })
expect(mockCustomApi.createNamespacedCustomObject).not.toHaveBeenCalledWith(
expect.objectContaining({ plural: 'contexts' })
)
} finally {
- errorSpy.mockRestore()
+ errorLog.mockRestore()
}
})
@@ -2447,6 +2489,7 @@ describe('delegateTransportWorkloads', () => {
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2509,6 +2552,7 @@ describe('delegateTransportWorkloads', () => {
.mockRejectedValueOnce({ code: 404 }),
}
const mockCoreApi = {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
}
const deps: DelegationDeps = {
@@ -2742,7 +2786,7 @@ describe('waitForExternalEgressReady', () => {
}
await expect(
- waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)
+ waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 'test-recipe', 1000)
).resolves.toEqual({ ready: true, pending: [], failed: [] })
})
@@ -2766,7 +2810,13 @@ describe('waitForExternalEgressReady', () => {
coreApi: {} as any,
}
- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)
+ const result = await waitForExternalEgressReady(
+ deps,
+ ['web-search'],
+ 'mcp-server',
+ 'test-recipe',
+ 1
+ )
expect(result.ready).toBe(false)
expect(result.pending).toEqual(['web-search'])
@@ -2794,7 +2844,13 @@ describe('waitForExternalEgressReady', () => {
coreApi: {} as any,
}
- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)
+ const result = await waitForExternalEgressReady(
+ deps,
+ ['web-search'],
+ 'mcp-server',
+ 'test-recipe',
+ 1
+ )
expect(result.ready).toBe(false)
expect(result.pending).toEqual(['web-search'])
@@ -2822,7 +2878,13 @@ describe('waitForExternalEgressReady', () => {
coreApi: {} as any,
}
- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1)
+ const result = await waitForExternalEgressReady(
+ deps,
+ ['web-search'],
+ 'mcp-server',
+ 'test-recipe',
+ 1
+ )
expect(result).toEqual({ ready: false, pending: ['web-search'], failed: [] })
})
@@ -2847,7 +2909,13 @@ describe('waitForExternalEgressReady', () => {
coreApi: {} as any,
}
- const result = await waitForExternalEgressReady(deps, ['web-search'], 'mcp-server', 1000)
+ const result = await waitForExternalEgressReady(
+ deps,
+ ['web-search'],
+ 'mcp-server',
+ 'test-recipe',
+ 1000
+ )
expect(result).toEqual({
ready: false,
pending: ['web-search'],
@@ -2878,7 +2946,7 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {
coreApi: {} as any,
}
- const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)
+ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)
expect(result.ready).toBe(false)
expect(result.pending).toEqual(['srv-a'])
@@ -2897,7 +2965,7 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {
coreApi: {} as any,
}
- const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)
+ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)
expect(result.ready).toBe(false)
expect(result.pending).toEqual(['srv-a'])
@@ -2919,7 +2987,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {
coreApi: {} as any,
}
- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({
+ await expect(
+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)
+ ).resolves.toEqual({
ready: true,
pending: [],
})
@@ -2935,7 +3005,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {
coreApi: {} as any,
}
- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({
+ await expect(
+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)
+ ).resolves.toEqual({
ready: true,
pending: [],
})
@@ -2949,7 +3021,9 @@ describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {
coreApi: {} as any,
}
- await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({
+ await expect(
+ waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 'test-recipe', 25)
+ ).resolves.toEqual({
ready: true,
pending: [],
})
@@ -2996,6 +3070,7 @@ describe('Issue #637 — transport Secret ownership gate', () => {
...overrides?.customApi,
} as unknown as DelegationDeps['customApi'],
coreApi: {
+ readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedService: vi.fn().mockResolvedValue({}),
deleteNamespacedService: vi.fn().mockResolvedValue({}),
...overrides?.coreApi,
@@ -3140,3 +3215,119 @@ describe('Issue #637 — transport Secret ownership gate', () => {
})
})
})
+
+// ─── Transport Service read-first apply ──────────────────────────────────────
+
+describe('transport Service read-first apply', () => {
+ type LiveService = {
+ metadata: { name: string; resourceVersion?: string; annotations?: Record<string, string> }
+ spec: { clusterIP?: string; ports?: Array<{ port: number }> }
+ }
+ const NS = 'mcp-server'
+ const NAME = 'test-recipe-redis-mcp'
+ const LIVE_CLUSTER_IP = '10.0.0.9'
+
+ // Stateful Service double: an absent object reads as 404, a create of a
+ // present one is a 409, the apiserver assigns clusterIP on create, and a
+ // replace must carry the live resourceVersion.
+ function serviceDeps() {
+ const live = new Map<string, LiveService>()
+ const coreApi = {
+ readNamespacedService: vi.fn(async ({ name }: { name: string }) => {
+ const found = live.get(name)
+ if (!found) throw { code: 404 }
+ return structuredClone(found)
+ }),
+ createNamespacedService: vi.fn(async ({ body }: { body: LiveService }) => {
+ if (live.has(body.metadata.name)) throw { code: 409 }
+ const stored = structuredClone(body)
+ stored.metadata.resourceVersion = '1'
+ stored.spec.clusterIP = LIVE_CLUSTER_IP
+ live.set(body.metadata.name, stored)
+ return {}
+ }),
+ replaceNamespacedService: vi.fn(
+ async ({ name, body }: { name: string; body: LiveService }) => {
+ const current = live.get(name)
+ if (!current || body.metadata.resourceVersion !== current.metadata.resourceVersion) {
+ throw { code: 409 }
+ }
+ const stored = structuredClone(body)
+ stored.metadata.resourceVersion = String(Number(current.metadata.resourceVersion) + 1)
+ live.set(name, stored)
+ return {}
+ }
+ ),
+ }
+ const deps: DelegationDeps = {
+ customApi: {
+ createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),
+ } as unknown as DelegationDeps['customApi'],
+ coreApi: coreApi as unknown as DelegationDeps['coreApi'],
+ }
+ const clear = () => {
+ coreApi.readNamespacedService.mockClear()
+ coreApi.createNamespacedService.mockClear()
+ coreApi.replaceNamespacedService.mockClear()
+ }
+ return { deps, coreApi, clear, live: () => live.get(NAME) }
+ }
+
+ it('reads an absent transport Service first, then sends exactly one POST and no PUT', async () => {
+ const { deps, coreApi } = serviceDeps()
+
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+
+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })
+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)
+ expect(coreApi.readNamespacedService.mock.invocationCallOrder[0]).toBeLessThan(
+ coreApi.createNamespacedService.mock.invocationCallOrder[0]
+ )
+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('sends neither POST nor PUT for an unchanged transport Service', async () => {
+ const { deps, coreApi, clear } = serviceDeps()
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+ clear()
+
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+
+ // Witness: the gate read the live Service before deciding not to write.
+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(1)
+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({ name: NAME, namespace: NS })
+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ it('replaces a changed transport Service once, keeping the live clusterIP', async () => {
+ const { deps, coreApi, clear, live } = serviceDeps()
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+ clear()
+ const changed = makeRecipe()
+ changed.spec.workloads![0].port = 3100
+
+ await delegateTransportWorkloads(deps, changed, NS, new Map())
+
+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)
+ expect(live()!.spec.clusterIP).toBe(LIVE_CLUSTER_IP)
+ expect(live()!.spec.ports![0].port).toBe(3100)
+ })
+
+ it('skips the write when another writer created the desired Service between read and POST', async () => {
+ const { deps, coreApi, clear } = serviceDeps()
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+ clear()
+ // The gate read misses the object the other writer is about to create.
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
+
+ await delegateTransportWorkloads(deps, makeRecipe(), NS, new Map())
+
+ expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(1)
+ // Witness: the conflict re-read the winner before deciding not to write.
+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)
+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+})
diff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.ts
index 305370cba..efc6e9352 100644
--- a/workflow-recipes/src/reconciler/mcpDelegation.ts
+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts
@@ -21,6 +21,7 @@ import {
} from '@clerum/workflow-runtime-core'
import { loadConfig } from '../config'
import { createLogger } from '../observability/logger'
+import type { Logger } from '../observability/logger'
import { WorkflowRecipeCRD, WorkloadDef } from '../types'
import { CRD_GROUP, CRD_VERSION } from './crdConstants'
import { getErrorCode } from './k8sErrors'
@@ -581,30 +582,55 @@ export function buildTransportService(
// ─── K8s API Operations ──────────────────────────────────────────────
-/** Create or update a ClusterIP Service. Idempotent (409 → read + replace preserving clusterIP). */
+/**
+ * Create or update a ClusterIP Service, reading first. The desired manifest is
+ * stamped with its spec hash; a live Service carrying the same hash is left
+ * alone, so a steady-state reconcile sends neither POST nor PUT. An absent
+ * Service is created; a 409 on that create means another writer got there
+ * between the read and the POST, so the gate is re-evaluated against the object
+ * it wrote. A changed Service is replaced carrying the live resourceVersion and
+ * clusterIP (spec.clusterIP is immutable once assigned).
+ */
async function ensureTransportService(
deps: DelegationDeps,
svc: k8s.V1Service,
- namespace: string
+ namespace: string,
+ log: Logger
): Promise<void> {
const name = svc.metadata!.name!
+ stampSpecHash(svc)
+
+ let existing: k8s.V1Service | null
try {
- await deps.coreApi.createNamespacedService({ namespace, body: svc })
- console.log(`[MCP-Delegation] Created transport Service "${name}"`)
+ existing = await deps.coreApi.readNamespacedService({ name, namespace })
} catch (error) {
- if (getErrorCode(error) === 409) {
- const existing = await deps.coreApi.readNamespacedService({ name, namespace })
- const updatedSvc: k8s.V1Service = {
- ...svc,
- metadata: { ...svc.metadata, resourceVersion: existing.metadata?.resourceVersion },
- spec: { ...svc.spec, clusterIP: existing.spec?.clusterIP },
- }
- await deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
- console.log(`[MCP-Delegation] Updated transport Service "${name}"`)
- } else {
- throw error
+ if (getErrorCode(error) !== 404) throw error
+ existing = null
+ }
+
+ if (!existing) {
+ try {
+ await deps.coreApi.createNamespacedService({ namespace, body: svc })
+ log.info('Created transport Service', { service: name })
+ return
+ } catch (error) {
+ if (getErrorCode(error) !== 409) throw error
}
+ existing = await deps.coreApi.readNamespacedService({ name, namespace })
}
+
+ if (specHashUnchanged(svc, existing)) {
+ log.debug('Transport Service unchanged; skipping update', { service: name })
+ return
+ }
+
+ const updatedSvc: k8s.V1Service = {
+ ...svc,
+ metadata: { ...svc.metadata, resourceVersion: existing.metadata?.resourceVersion },
+ spec: { ...svc.spec, clusterIP: existing.spec?.clusterIP },
+ }
+ await deps.coreApi.replaceNamespacedService({ name, namespace, body: updatedSvc })
+ log.info('Updated transport Service', { service: name })
}
/** Create or update an McpServer CRD. Idempotent (409 → replace).
@@ -616,7 +642,8 @@ async function ensureTransportService(
async function ensureMcpServer(
deps: DelegationDeps,
manifest: Record<string, unknown>,
- namespace: string
+ namespace: string,
+ log: Logger
): Promise<void> {
const meta = manifest.metadata as { name: string; labels?: Record<string, string> }
const desiredRecipeLabel = meta.labels?.['clerum.io/recipe']
@@ -753,9 +780,10 @@ async function ensureMcpServer(
isRecipeOwned === desiredRecipeLabel &&
mcpServerSpecMatches(latest.spec, manifest.spec)
) {
- console.warn(
- `[MCP-Delegation] McpServer "${meta.name}" kept existing recipe-owned object after ` +
- `${MCPSERVER_REPLACE_CONFLICT_RETRIES} replace conflict retries; existing spec already matches desired state.`
+ log.warn(
+ 'McpServer kept existing recipe-owned object after replace conflict retries; ' +
+ 'existing spec already matches desired state',
+ { mcpServer: meta.name, retries: MCPSERVER_REPLACE_CONFLICT_RETRIES }
)
return false
}
@@ -774,9 +802,9 @@ async function ensureMcpServer(
const existing = await readExisting()
const replaced = await replaceFromExisting(existing)
if (replaced) {
- console.log(`[MCP-Delegation] Updated McpServer "${meta.name}"`)
+ log.info('Updated McpServer', { mcpServer: meta.name })
} else {
- console.log(`[MCP-Delegation] McpServer "${meta.name}" unchanged; skipping update`)
+ log.info('McpServer unchanged; skipping update', { mcpServer: meta.name })
}
} catch (error) {
if (getErrorCode(error) === 404) {
@@ -789,17 +817,17 @@ async function ensureMcpServer(
plural: MCPSERVER_PLURAL,
body: manifest,
})
- console.log(`[MCP-Delegation] Created McpServer "${meta.name}"`)
+ log.info('Created McpServer', { mcpServer: meta.name })
} catch (createError) {
if (getErrorCode(createError) === 409) {
const latest = await readExisting()
const replaced = await replaceFromExisting(latest)
if (replaced) {
- console.log(`[MCP-Delegation] Updated McpServer "${meta.name}" after create conflict`)
+ log.info('Updated McpServer after create conflict', { mcpServer: meta.name })
} else {
- console.log(
- `[MCP-Delegation] McpServer "${meta.name}" unchanged after create conflict; skipping update`
- )
+ log.info('McpServer unchanged after create conflict; skipping update', {
+ mcpServer: meta.name,
+ })
}
} else {
throw createError
@@ -817,7 +845,8 @@ async function ensureMcpServer(
async function deleteMcpServer(
deps: DelegationDeps,
name: string,
- namespace: string
+ namespace: string,
+ log: Logger
): Promise<void> {
try {
await deps.customApi.deleteNamespacedCustomObject({
@@ -827,10 +856,10 @@ async function deleteMcpServer(
plural: MCPSERVER_PLURAL,
name,
})
- console.log(`[MCP-Delegation] Deleted McpServer "${name}"`)
+ log.info('Deleted McpServer', { mcpServer: name })
} catch (error) {
if (getErrorCode(error) === 404) {
- console.log(`[MCP-Delegation] McpServer "${name}" already gone`)
+ log.info('McpServer already gone', { mcpServer: name })
} else {
throw error
}
@@ -854,10 +883,11 @@ export async function deleteTransportDelegation(
): Promise<void> {
if (!workload.transport) return
const name = mcpServerName(recipe.metadata.name, workload.id, recipe)
- await deleteMcpServer(deps, name, namespace)
+ const log = createLogger('wrc', recipe.metadata.name)
+ await deleteMcpServer(deps, name, namespace, log)
try {
await deps.coreApi.deleteNamespacedService({ name, namespace })
- console.log(`[MCP-Delegation] Deleted transport Service "${name}" (ownership revoked)`)
+ log.info('Deleted transport Service (ownership revoked)', { service: name })
} catch (error) {
if (getErrorCode(error) !== 404) throw error
}
@@ -1006,10 +1036,7 @@ export async function ensureRecipeContext(
throw new Error(`failed to update Context "${contextName}" after conflict retries`)
}
- // One mechanism for all three outcomes of this one decision. The skip branch
- // introduced `createLogger` while its siblings stayed on `console.log`, which
- // left the same event reported two ways (#568 review, R1-L5). Scoped to this
- // function: the file-wide console -> logger migration is #569.
+ // One mechanism for all three outcomes of this one decision (#568 review, R1-L5).
const log = createLogger('wrc', recipeName)
try {
@@ -1087,14 +1114,15 @@ export async function preDeployMcpServers(
namespace: string,
secretAccess: ReadonlyMap<string, SecretAccess>
): Promise<string[]> {
+ const log = createLogger('wrc', recipe.metadata.name)
const transportWorkloads = (recipe.spec.workloads ?? [])
.filter(w => !!w.transport)
.filter(workload => {
if (transportWorkloadSecretDenied(workload, secretAccess)) {
- console.error(
- `[MCP-Delegation] Issue #637: refusing to pre-deploy transport workload "${workload.id}" ` +
- `in recipe "${recipe.metadata.name}" — it references a Secret it does not own ` +
- `(denied/unverified). No McpServer CRD or Service will be created.`
+ log.error(
+ 'Issue #637: refusing to pre-deploy transport workload that references a Secret ' +
+ 'it does not own (denied/unverified). No McpServer CRD or Service will be created.',
+ { workloadId: workload.id }
)
return false
}
@@ -1117,10 +1145,10 @@ export async function preDeployMcpServers(
// Also create the transport Service (needed for DNS resolution in McpServer URL)
const svc = buildTransportService(workload, recipe, namespace)
if (svc) {
- await ensureTransportService(deps, svc, namespace)
+ await ensureTransportService(deps, svc, namespace, log)
}
- await ensureMcpServer(deps, manifest, namespace)
+ await ensureMcpServer(deps, manifest, namespace, log)
return mcpServerName(recipe.metadata.name, workload.id, recipe)
})
)
@@ -1132,9 +1160,7 @@ export async function preDeployMcpServers(
preDeployed.push(r.value)
} else if (r.status === 'rejected') {
const workloadId = transportWorkloads[i].id
- console.warn(
- `[MCP-Delegation] Pre-deploy failed for workload "${workloadId}": ${String(r.reason)}`
- )
+ log.warn('Pre-deploy failed for workload', { workloadId, error: r.reason })
errors.push({ workloadId, error: r.reason })
}
}
@@ -1195,10 +1221,12 @@ export async function waitForNetworkReady(
deps: DelegationDeps,
serverNames: string[],
namespace: string,
+ recipeName: string,
timeoutMs: number = NETWORK_READY_TIMEOUT_MS
): Promise<{ ready: boolean; pending: string[] }> {
if (serverNames.length === 0) return { ready: true, pending: [] }
+ const log = createLogger('wrc', recipeName)
const deadline = Date.now() + timeoutMs
const pending = new Set(serverNames)
@@ -1238,7 +1266,7 @@ export async function waitForNetworkReady(
for (const { name, ready } of results) {
if (ready) {
pending.delete(name)
- console.log(`[MCP-Delegation] Network ready for "${name}"`)
+ log.info('Network ready', { mcpServer: name })
}
}
@@ -1248,9 +1276,10 @@ export async function waitForNetworkReady(
}
if (pending.size > 0) {
- console.warn(
- `[MCP-Delegation] Network ready timeout (${timeoutMs}ms) for: ${[...pending].join(', ')}. ` +
- `This generic readiness check is not used to unblock workloads with external egress.`
+ log.warn(
+ 'Network ready timeout. This generic readiness check is not used to unblock ' +
+ 'workloads with external egress.',
+ { timeoutMs, pending: [...pending].join(', ') }
)
}
@@ -1277,6 +1306,7 @@ export async function waitForExternalEgressReady(
deps: DelegationDeps,
serverNames: string[],
namespace: string,
+ recipeName: string,
timeoutMs: number = NETWORK_READY_TIMEOUT_MS
): Promise<{
ready: boolean
@@ -1285,6 +1315,7 @@ export async function waitForExternalEgressReady(
}> {
if (serverNames.length === 0) return { ready: true, pending: [], failed: [] }
+ const log = createLogger('wrc', recipeName)
const deadline = Date.now() + timeoutMs
const pending = new Set(serverNames)
const failed = new Map<string, string>()
@@ -1335,7 +1366,7 @@ export async function waitForExternalEgressReady(
for (const result of results) {
if (result.status === 'True') {
pending.delete(result.name)
- console.log(`[MCP-Delegation] External egress ready for "${result.name}"`)
+ log.info('External egress ready', { mcpServer: result.name })
} else if (result.status === 'False') {
if (!result.reason || !RETRYABLE_EXTERNAL_EGRESS_REASONS.has(result.reason)) {
failed.set(result.name, result.message ?? `${EXTERNAL_EGRESS_READY_CONDITION} is False`)
@@ -1350,12 +1381,9 @@ export async function waitForExternalEgressReady(
const failedList = [...failed.entries()].map(([name, message]) => ({ name, message }))
if (pending.size > 0 || failedList.length > 0) {
- console.warn(
- `[MCP-Delegation] External egress readiness not achieved for: ${[
- ...pending,
- ...failed.keys(),
- ].join(', ')}`
- )
+ log.warn('External egress readiness not achieved', {
+ pending: [...pending, ...failed.keys()].join(', '),
+ })
}
return {
@@ -1400,14 +1428,15 @@ export async function delegateTransportWorkloads(
namespace: string,
secretAccess: ReadonlyMap<string, SecretAccess>
): Promise<string[]> {
+ const log = createLogger('wrc', recipe.metadata.name)
const transportWorkloads = (recipe.spec.workloads ?? [])
.filter(w => !!w.transport)
.filter(workload => {
if (transportWorkloadSecretDenied(workload, secretAccess)) {
- console.error(
- `[MCP-Delegation] Issue #637: refusing to delegate transport workload "${workload.id}" ` +
- `in recipe "${recipe.metadata.name}" — it references a Secret it does not own ` +
- `(denied/unverified). Its McpServer CRD will not be (re)created.`
+ log.error(
+ 'Issue #637: refusing to delegate transport workload that references a Secret ' +
+ 'it does not own (denied/unverified). Its McpServer CRD will not be (re)created.',
+ { workloadId: workload.id }
)
return false
}
@@ -1422,13 +1451,13 @@ export async function delegateTransportWorkloads(
// 1. Create transport Service
const svc = buildTransportService(workload, recipe, namespace)
if (svc) {
- await ensureTransportService(deps, svc, namespace)
+ await ensureTransportService(deps, svc, namespace, log)
}
// 2. Create McpServer CRD
const manifest = buildMcpServerManifest(workload, recipe, namespace, secretAccess)
if (manifest) {
- await ensureMcpServer(deps, manifest, namespace)
+ await ensureMcpServer(deps, manifest, namespace, log)
}
return serverName
@@ -1442,7 +1471,7 @@ export async function delegateTransportWorkloads(
delegated.push(r.value)
} else {
const workloadId = transportWorkloads[i].id
- console.error(`[MCP-Delegation] Failed to delegate workload "${workloadId}":`, r.reason)
+ log.error('Failed to delegate workload', { workloadId, error: r.reason })
errors.push({ workloadId, error: r.reason })
}
}
@@ -1497,6 +1526,7 @@ export async function cleanupDelegation(
recipe: WorkflowRecipeCRD,
namespace: string
): Promise<void> {
+ const log = createLogger('wrc', recipe.metadata.name)
const transportWorkloads = (recipe.spec.workloads ?? []).filter(w => !!w.transport)
const cleanupFailures: string[] = []
@@ -1506,15 +1536,15 @@ export async function cleanupDelegation(
const serverName = mcpServerName(recipe.metadata.name, workload.id, recipe)
// 1. Delete McpServer CRD (ownerRef GC also handles this)
- await deleteMcpServer(deps, serverName, namespace)
+ await deleteMcpServer(deps, serverName, namespace, log)
// 2. Delete transport Service (belt-and-suspenders)
try {
await deps.coreApi.deleteNamespacedService({ name: serverName, namespace })
- console.log(`[MCP-Delegation] Deleted transport Service "${serverName}"`)
+ log.info('Deleted transport Service', { service: serverName })
} catch (error) {
if (getErrorCode(error) === 404) {
- console.log(`[MCP-Delegation] Transport Service "${serverName}" already gone`)
+ log.info('Transport Service already gone', { service: serverName })
} else {
throw error
}
@@ -1577,10 +1607,10 @@ export async function cleanupDelegation(
if (!removed) {
throw new Error(`failed to update Context "${recipeContextName}" after conflict retries`)
}
- console.log(`[MCP-Delegation] Removed recipe servers from Context "${recipeContextName}"`)
+ log.info('Removed recipe servers from Context', { contextName: recipeContextName })
} catch (error) {
if (getErrorCode(error) === 404) {
- console.log(`[MCP-Delegation] Explicit Context "${recipeContextName}" already gone`)
+ log.info('Explicit Context already gone', { contextName: recipeContextName })
} else {
cleanupFailures.push(
`Context "${recipeContextName}": ${
@@ -1598,10 +1628,10 @@ export async function cleanupDelegation(
plural: CONTEXT_PLURAL,
name: recipeContextName,
})
- .then(() => console.log(`[MCP-Delegation] Deleted per-recipe Context "${recipeContextName}"`))
+ .then(() => log.info('Deleted per-recipe Context', { contextName: recipeContextName }))
.catch(error => {
if (getErrorCode(error) === 404) {
- console.log(`[MCP-Delegation] Per-recipe Context "${recipeContextName}" already gone`)
+ log.info('Per-recipe Context already gone', { contextName: recipeContextName })
} else {
cleanupFailures.push(
`Context "${recipeContextName}": ${
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 282666034..c5b9c87e0 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -14,7 +14,10 @@ import { defaultFqdnLookup } from './fqdnResolver'
import { isRetryableInfraError } from './k8sErrors'
import type { NetworkPolicyFamily } from './networkPolicyConvergence'
import * as brokerIssuer from './oauthBrokerTokenIssuerClient'
-import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'
+import {
+ PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
+ PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE,
+} from './pluginWorkloadSdkValidator'
import {
buildUiIngressNetworkPolicy,
buildWorkloadIngressNetworkPolicy,
@@ -208,6 +211,121 @@ function egressStateAnnotations(
}
}
+type LiveManifest = { metadata: { name: string } & Record<string, unknown> } & Record<
+ string,
+ unknown
+>
+
+/**
+ * Stateful apiserver double for one spec-hash-gated kind. `createOrReplace`
+ * reads before it writes, so an object this test never created must read as a
+ * 404; one it created or replaced reads back as written, carrying `status` —
+ * the same "ready" status the observation path used to get from a static read.
+ * `admit` models fields the apiserver assigns on create (a Service's clusterIP).
+ */
+function installLiveStore(
+ api: {
+ read: ReturnType<typeof vi.fn>
+ create: ReturnType<typeof vi.fn>
+ replace: ReturnType<typeof vi.fn>
+ },
+ status: Record<string, unknown>,
+ admit?: (stored: LiveManifest) => void
+): void {
+ const live = new Map<string, LiveManifest>()
+ const key = (namespace: string, name: string) => `${namespace}/${name}`
+ api.read
+ .mockReset()
+ .mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => {
+ const found = live.get(key(namespace, name))
+ if (!found) throw { code: 404 }
+ return structuredClone(found)
+ })
+ api.create
+ .mockReset()
+ .mockImplementation(async ({ namespace, body }: { namespace: string; body: LiveManifest }) => {
+ const k = key(namespace, body.metadata.name)
+ if (live.has(k)) throw { code: 409 }
+ const stored = structuredClone(body)
+ stored.metadata = { ...stored.metadata, resourceVersion: '1', generation: 1 }
+ admit?.(stored)
+ live.set(k, { ...stored, status: structuredClone(status) })
+ return {}
+ })
+ api.replace
+ .mockReset()
+ .mockImplementation(
+ async ({
+ name,
+ namespace,
+ body,
+ }: {
+ name: string
+ namespace: string
+ body: LiveManifest
+ }) => {
+ const k = key(namespace, name)
+ const current = live.get(k)
+ // A test that overrides `read` supplies the live object itself, so an
+ // object the store never saw is accepted here rather than 404'd.
+ if (current && body.metadata.resourceVersion !== current.metadata.resourceVersion) {
+ throw { code: 409 }
+ }
+ const stored = structuredClone(body)
+ stored.metadata = {
+ ...stored.metadata,
+ resourceVersion: String(Number(current?.metadata.resourceVersion ?? '1') + 1),
+ generation: 1,
+ }
+ live.set(k, { ...stored, status: structuredClone(status) })
+ return {}
+ }
+ )
+}
+
+const LIVE_CLUSTER_IP = '10.0.0.7'
+
+function installLiveServiceAndConfigMapStores(): void {
+ installLiveStore(
+ {
+ read: mockCoreApi.readNamespacedService,
+ create: mockCoreApi.createNamespacedService,
+ replace: mockCoreApi.replaceNamespacedService,
+ },
+ {},
+ stored => {
+ stored.spec = { ...(stored.spec as Record<string, unknown>), clusterIP: LIVE_CLUSTER_IP }
+ }
+ )
+ installLiveStore(
+ {
+ read: mockCoreApi.readNamespacedConfigMap,
+ create: mockCoreApi.createNamespacedConfigMap,
+ replace: mockCoreApi.replaceNamespacedConfigMap,
+ },
+ {}
+ )
+}
+
+// Per-object write counts: one reconcile writes several objects of the same
+// kind, so a bare call count cannot say which of them was written.
+function postsOf(create: ReturnType<typeof vi.fn>, name: string): number {
+ return create.mock.calls.filter(
+ ([args]) => (args as { body: LiveManifest }).body.metadata.name === name
+ ).length
+}
+
+function putsOf(replace: ReturnType<typeof vi.fn>, name: string): number {
+ return replace.mock.calls.filter(([args]) => (args as { name: string }).name === name).length
+}
+
+function readsOf(read: ReturnType<typeof vi.fn>, name: string, namespace: string): number {
+ return read.mock.calls.filter(([args]) => {
+ const a = args as { name: string; namespace: string }
+ return a.name === name && a.namespace === namespace
+ }).length
+}
+
describe('WorkflowRecipeReconciler', () => {
let reconciler: WorkflowRecipeReconciler
@@ -215,12 +333,14 @@ describe('WorkflowRecipeReconciler', () => {
vi.clearAllMocks()
mockVerifyWorkflowRunProvenance.mockResolvedValue('verified')
process.env.CLERUM_NETWORK_POLICY_ENFORCEMENT_CONFIRMED = 'true'
- mockAppsApi.readNamespacedDeployment.mockReset()
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1', generation: 1 },
- spec: { replicas: 1 },
- status: { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 },
- })
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedDeployment,
+ create: mockAppsApi.createNamespacedDeployment,
+ replace: mockAppsApi.replaceNamespacedDeployment,
+ },
+ { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 1, availableReplicas: 1 }
+ )
mockAppsApi.readNamespacedStatefulSet.mockReset()
mockAppsApi.readNamespacedStatefulSet.mockResolvedValue({
metadata: { resourceVersion: '1' },
@@ -229,22 +349,31 @@ describe('WorkflowRecipeReconciler', () => {
})
mockAppsApi.patchNamespacedStatefulSet.mockReset()
mockAppsApi.patchNamespacedStatefulSet.mockResolvedValue({})
- mockAppsApi.readNamespacedDaemonSet.mockReset()
- mockAppsApi.readNamespacedDaemonSet.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { desiredNumberScheduled: 1, numberReady: 1 },
- })
- mockBatchApi.readNamespacedCronJob.mockReset()
- mockBatchApi.readNamespacedCronJob.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- spec: { suspend: false },
- })
- mockBatchApi.readNamespacedJob.mockReset()
- mockBatchApi.readNamespacedJob.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- spec: { completions: 1 },
- status: { succeeded: 1 },
- })
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedDaemonSet,
+ create: mockAppsApi.createNamespacedDaemonSet,
+ replace: mockAppsApi.replaceNamespacedDaemonSet,
+ },
+ { desiredNumberScheduled: 1, numberReady: 1 }
+ )
+ installLiveStore(
+ {
+ read: mockBatchApi.readNamespacedCronJob,
+ create: mockBatchApi.createNamespacedCronJob,
+ replace: mockBatchApi.replaceNamespacedCronJob,
+ },
+ {}
+ )
+ installLiveStore(
+ {
+ read: mockBatchApi.readNamespacedJob,
+ create: mockBatchApi.createNamespacedJob,
+ replace: mockBatchApi.replaceNamespacedJob,
+ },
+ { succeeded: 1 }
+ )
+ installLiveServiceAndConfigMapStores()
mockCoreApi.readNamespacedPersistentVolumeClaim.mockReset()
mockCoreApi.readNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 404 })
mockCustomApi.createNamespacedCustomObject.mockReset()
@@ -273,16 +402,16 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
({ name }: { name: string }) => {
if (!name.endsWith('-coordinator-to-gfs')) return Promise.reject({ code: 404 })
+ // The live policy as WRC last wrote it: owned and converged, so a
+ // read-first apply has nothing to write. Tests about an absent policy
+ // say so explicitly.
+ const converged = buildCoordinatorGfsNetworkPolicy({
+ recipeName: name.replace(/-coordinator-to-gfs$/, ''),
+ sandboxNamespace: 'sandbox-recipes',
+ })
return Promise.resolve({
- metadata: {
- name,
- uid: `uid-${name}`,
- resourceVersion: '1',
- labels: {
- 'clerum.io/managed-by': 'wrc',
- 'clerum.io/recipe': name.replace(/-coordinator-to-gfs$/, ''),
- },
- },
+ ...converged,
+ metadata: { ...converged.metadata, name, uid: `uid-${name}`, resourceVersion: '1' },
})
}
)
@@ -500,10 +629,15 @@ describe('WorkflowRecipeReconciler', () => {
it('preserves ordinary Pod-only downgrade and recovery when no admission is pending', async () => {
const recipe = makeRecipe({ status: { phase: 'degraded' } })
- mockAppsApi.readNamespacedDeployment.mockResolvedValueOnce({
- spec: { replicas: 1 },
- status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },
- })
+ mockAppsApi.readNamespacedDeployment
+ .mockResolvedValueOnce({
+ spec: { replicas: 1 },
+ status: { readyReplicas: 0, updatedReplicas: 0, availableReplicas: 0 },
+ })
+ .mockResolvedValueOnce({
+ spec: { replicas: 1 },
+ status: { readyReplicas: 1, updatedReplicas: 1, availableReplicas: 1 },
+ })
expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('degraded')
expect((await reconciler.observeCurrentWorkloadStatus(recipe)).phase).toBe('active')
})
@@ -972,9 +1106,9 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!
address = '93.184.216.21'
if (failure === 'read-disappearance') {
- // The workload lane is read-first: start absent so its create can
- // race with another creator, whose winner then disappears.
- if (lane === 'workload') live.delete(name)
+ // Both lanes are read-first: start absent so the create can race
+ // with another creator, whose winner then disappears.
+ live.delete(name)
mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(async request => {
if (request.body.metadata.name === name) {
live.delete(name)
@@ -1284,6 +1418,68 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ // awaiting_policy waits for an operator grant. The grant arrives by event
+ // (handleGrantUpdateNotification) or by the 30s credential-refresh floor, so a
+ // fixed 5s poll cannot make the state advance. It must keep requeueing, but on
+ // the transient backoff path rather than the fixed progress interval.
+ describe('SDK-only recipe awaiting operator policy', () => {
+ const reconcileAwaitingPolicy = async () => {
+ const reconcilePluginWorkloadSdkOnly = vi.fn().mockResolvedValue({
+ phase: 'awaiting_policy',
+ message: 'operator policy pending (policy_not_ready)',
+ })
+ ;(
+ reconciler as unknown as {
+ config: { pluginWorkloadSdkEnabled: boolean }
+ workflowReconciler: {
+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly
+ }
+ }
+ ).config.pluginWorkloadSdkEnabled = true
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcilePluginWorkloadSdkOnly: typeof reconcilePluginWorkloadSdkOnly
+ }
+ }
+ ).workflowReconciler = { reconcilePluginWorkloadSdkOnly }
+
+ const recipe = makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ workloads: [{ id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['app'] },
+ },
+ })
+ const result = await reconciler.reconcile(recipe)
+
+ // Liveness witness: the awaiting_policy branch ran, and the watcher's
+ // projection of this result publishes PluginWorkloadSdkPolicyPending=True.
+ expect(reconcilePluginWorkloadSdkOnly).toHaveBeenCalledTimes(1)
+ expect(result.pluginWorkloadSdkPolicyPending).toBe(true)
+ expect(result.message).toBe('operator policy pending (policy_not_ready)')
+ expect(reconciler.projectPluginWorkloadSdk(recipe, result).conditions).toContainEqual(
+ expect.objectContaining({
+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
+ status: 'True',
+ })
+ )
+ return result
+ }
+
+ it('does not requeue at a fixed interval', async () => {
+ const result = await reconcileAwaitingPolicy()
+
+ expect(result.requeueFixedInterval).not.toBe(true)
+ })
+
+ it('still requeues, on the transient backoff base', async () => {
+ const result = await reconcileAwaitingPolicy()
+
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ })
+ })
+
it('cleans SDK-only runtime resources after the capability is removed without invoking workflow reconciliation', async () => {
const cleanupPluginWorkloadSdk = vi.fn().mockResolvedValue(undefined)
const workflowReconcile = vi.fn()
@@ -2383,59 +2579,450 @@ describe('WorkflowRecipeReconciler', () => {
mockBatchApi.replaceNamespacedJob.mockReset().mockResolvedValue({})
})
- it('does NOT replace a Deployment whose spec-hash is unchanged', async () => {
+ it('replaces a Deployment when the desired spec-hash differs', async () => {
const recipe = makeRecipe()
const workload = recipe.spec.workloads![0]
- // First apply creates the object; capture the manifest WRC writes (carries the hash).
- let createdBody: { metadata: { annotations: Record<string, string> }; spec: unknown } | null =
- null
- mockAppsApi.createNamespacedDeployment.mockImplementation((args: { body: unknown }) => {
- createdBody = args.body as typeof createdBody
- return Promise.resolve({})
+ mockAppsApi.readNamespacedDeployment.mockResolvedValue({
+ metadata: { resourceVersion: '9', annotations: { [SPEC_HASH]: 'stale-or-foreign' } },
+ spec: {},
})
+ mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})
+
await (
reconciler as unknown as {
ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
}
).ensureDeployment(workload, recipe, 'minimal', {})
- expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()
- // Second apply: object already exists with the SAME hash → must skip the PUT.
- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },
- spec: createdBody!.spec,
+ // A present object with another hash is replaced directly: no POST probe.
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(
+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion
+ ).toBe('9')
+ })
+
+ describe('read-first apply', () => {
+ type Manifest = { metadata: { name: string; namespace: string; resourceVersion?: string } }
+ const ensureDeployment = (workload: unknown, recipe: unknown) =>
+ (
+ reconciler as unknown as {
+ ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ }
+ ).ensureDeployment(workload, recipe, 'minimal', {})
+ const ensureStatefulSet = (workload: unknown, recipe: unknown) =>
+ (
+ reconciler as unknown as {
+ ensureStatefulSet: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ }
+ ).ensureStatefulSet(workload, recipe, 'minimal', {})
+
+ // Stateful Deployment double: a read of an absent object is a 404, a
+ // create of a present one is a 409, and a replace must carry the live
+ // resourceVersion — the apiserver's own contract.
+ const deploymentStore = () => {
+ let live: Manifest | null = null
+ mockAppsApi.readNamespacedDeployment.mockImplementation(async () => {
+ if (!live) throw { code: 404 }
+ return clone(live)
+ })
+ mockAppsApi.createNamespacedDeployment.mockImplementation(
+ async ({ body }: { body: Manifest }) => {
+ if (live) throw { code: 409 }
+ live = { ...clone(body), metadata: { ...clone(body).metadata, resourceVersion: '1' } }
+ return {}
+ }
+ )
+ mockAppsApi.replaceNamespacedDeployment.mockImplementation(
+ async ({ body }: { body: Manifest }) => {
+ expect(body.metadata.resourceVersion).toBe(live?.metadata.resourceVersion)
+ live = clone(body)
+ return {}
+ }
+ )
+ return { live: () => live }
+ }
+
+ it('reads an absent Deployment first, then sends exactly one POST and no PUT', async () => {
+ const recipe = makeRecipe()
+ const store = deploymentStore()
+
+ await ensureDeployment(recipe.spec.workloads![0], recipe)
+
+ const created = store.live()!
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledWith({
+ name: created.metadata.name,
+ namespace: created.metadata.namespace,
+ })
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mockAppsApi.readNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(
+ mockAppsApi.createNamespacedDeployment.mock.invocationCallOrder[0]
+ )
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)
})
- mockAppsApi.replaceNamespacedDeployment.mockClear()
- await (
- reconciler as unknown as {
- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ it('sends neither POST nor PUT for a live Deployment whose spec-hash is unchanged', async () => {
+ const recipe = makeRecipe()
+ const store = deploymentStore()
+ await ensureDeployment(recipe.spec.workloads![0], recipe)
+ const live = store.live()!
+ mockAppsApi.readNamespacedDeployment.mockClear()
+ mockAppsApi.createNamespacedDeployment.mockClear()
+ mockAppsApi.replaceNamespacedDeployment.mockClear()
+
+ await ensureDeployment(recipe.spec.workloads![0], recipe)
+
+ // Witness: the gate read the live object before deciding not to write.
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledWith({
+ name: live.metadata.name,
+ namespace: live.metadata.namespace,
+ })
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(0)
+ })
+
+ // The POST is mocked to 409 only so that the pre-read-first code reaches
+ // the gate at all; with the read first, it must never be sent.
+ it('propagates an unclassifiable gate-read error and writes nothing', async () => {
+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
+ const recipe = makeRecipe({ spec: { workloads: [workload] } })
+ const boom = new Error('boom')
+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(boom)
+ mockCoreApi.createNamespacedService.mockRejectedValue({ code: 409 })
+
+ const outcome = ensureStatefulSet(workload, recipe)
+
+ await expect(outcome).rejects.toBe(boom)
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalledTimes(1)
+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ })
+
+ // The same failure seen from reconcile(): the phase it lands in is part
+ // of the behaviour change, not an implementation detail.
+ it('degrades, not fails, a recipe whose gate read hits an unclassifiable error', async () => {
+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
+ const recipe = makeRecipe({ spec: { workloads: [workload] } })
+ const boom = new Error('boom')
+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(boom)
+
+ const result = await reconciler.reconcile(recipe)
+
+ expect(mockCoreApi.readNamespacedService).toHaveBeenCalled()
+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
+ // The workload fails like any other non-409 create failure, and the
+ // recipe degrades rather than latching at the terminal `failed`:
+ // degraded recipes are re-enqueued by the workload status refresh loop.
+ expect(result.phase).toBe('degraded')
+ expect(result.workloadStatuses).toEqual([
+ { id: 'db', phase: 'failed', ready: false, message: 'Error: boom' },
+ ])
+ })
+
+ it.each([
+ ['an apiserver 500', { code: 500 }],
+ [
+ 'a transport reset',
+ Object.assign(new Error('connect ECONNRESET 10.0.0.1:443'), { code: 'ECONNRESET' }),
+ ],
+ ])('falls through to the replace when the gate read fails with %s', async (_, failure) => {
+ const workload = { id: 'db', type: 'statefulset' as const, image: 'postgres:16' }
+ const recipe = makeRecipe({ spec: { workloads: [workload] } })
+ // The first pass creates the headless Service, so the replace has a
+ // live object to read back.
+ await ensureStatefulSet(workload, recipe)
+ mockCoreApi.createNamespacedService.mockClear()
+ mockCoreApi.replaceNamespacedService.mockClear()
+ mockCoreApi.readNamespacedService.mockRejectedValueOnce(failure)
+
+ await ensureStatefulSet(workload, recipe)
+
+ // Witness: the PUT went out, so the fall-through ran.
+ expect(mockCoreApi.replaceNamespacedService).toHaveBeenCalledTimes(1)
+ expect(mockCoreApi.createNamespacedService).toHaveBeenCalledTimes(0)
+ })
+ })
+
+ describe('read-first apply of Services and ConfigMaps', () => {
+ type CreateArgs = { namespace: string; body: LiveManifest }
+
+ const clearServiceAndConfigMapCalls = () => {
+ for (const fn of [
+ mockCoreApi.readNamespacedService,
+ mockCoreApi.createNamespacedService,
+ mockCoreApi.replaceNamespacedService,
+ mockCoreApi.readNamespacedConfigMap,
+ mockCoreApi.createNamespacedConfigMap,
+ mockCoreApi.replaceNamespacedConfigMap,
+ ]) {
+ fn.mockClear()
}
- ).ensureDeployment(workload, recipe, 'minimal', {})
+ }
+ const onlyPost = (create: ReturnType<typeof vi.fn>): CreateArgs => {
+ expect(create).toHaveBeenCalledTimes(1)
+ return create.mock.calls[0][0] as CreateArgs
+ }
+
+ it('sends neither POST nor PUT for an unchanged recipe-lane Service', async () => {
+ const recipe = makeRecipe()
+ await reconciler.reconcile(recipe)
+ const created = onlyPost(mockCoreApi.createNamespacedService)
+ const name = created.body.metadata.name
+ clearServiceAndConfigMapCalls()
+
+ await reconciler.reconcile(recipe)
+
+ // Witness: the gate read the live Service before deciding not to write.
+ expect(readsOf(mockCoreApi.readNamespacedService, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(0)
+ })
+
+ it('keeps the live clusterIP when a changed recipe-lane Service is replaced', async () => {
+ await reconciler.reconcile(makeRecipe())
+ const created = onlyPost(mockCoreApi.createNamespacedService)
+ const name = created.body.metadata.name
+ clearServiceAndConfigMapCalls()
+
+ await reconciler.reconcile(
+ makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 9090 },
+ ],
+ },
+ })
+ )
+
+ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(1)
+ const replaced = mockCoreApi.replaceNamespacedService.mock.calls[0][0] as {
+ body: { metadata: { resourceVersion?: string }; spec: { clusterIP?: string } }
+ }
+ expect(replaced.body.spec.clusterIP).toBe(LIVE_CLUSTER_IP)
+ expect(replaced.body.metadata.resourceVersion).toBe('1')
+ })
+
+ it('sends neither POST nor PUT for an unchanged workflow-lane Service', async () => {
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ steps: [{ id: 'run', run: snippetRun() }],
+ },
+ })
+ const deployWorkflowWorkloads = () =>
+ (
+ reconciler as unknown as {
+ deployWorkflowWorkloads: (r: WorkflowRecipeCRD, n: string) => Promise<unknown>
+ }
+ ).deployWorkflowWorkloads(recipe, recipe.metadata.name)
+ await deployWorkflowWorkloads()
+ const created = onlyPost(mockCoreApi.createNamespacedService)
+ const name = created.body.metadata.name
+ clearServiceAndConfigMapCalls()
+
+ await deployWorkflowWorkloads()
- expect(mockAppsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(readsOf(mockCoreApi.readNamespacedService, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedService, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedService, name)).toBe(0)
+ })
+
+ it('sends neither POST nor PUT for an unchanged resource ConfigMap', async () => {
+ const res = { id: 'cfg', type: 'configmap' as const, data: { k: 'v' } }
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ resources: [res],
+ },
+ })
+ const ensureConfigMap = () =>
+ (
+ reconciler as unknown as {
+ ensureConfigMap: (r: unknown, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ ).ensureConfigMap(res, recipe)
+ await ensureConfigMap()
+ const created = onlyPost(mockCoreApi.createNamespacedConfigMap)
+ const name = created.body.metadata.name
+ clearServiceAndConfigMapCalls()
+
+ await ensureConfigMap()
+
+ expect(readsOf(mockCoreApi.readNamespacedConfigMap, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedConfigMap, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedConfigMap, name)).toBe(0)
+ })
})
- it('replaces a Deployment when the desired spec-hash differs', async () => {
- const recipe = makeRecipe()
- const workload = recipe.spec.workloads![0]
+ describe('read-first apply of resource Secrets', () => {
+ type SecretManifest = LiveManifest & { data?: Record<string, string> }
+ type CreateArgs = { namespace: string; body: SecretManifest }
+ type SecretRes = {
+ id: string
+ type: 'secret'
+ data?: Record<string, string>
+ generateKeys?: string[]
+ }
- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '9', annotations: { [SPEC_HASH]: 'stale-or-foreign' } },
- spec: {},
+ // Local to this describe: many other tests override readNamespacedSecret
+ // with their own fixtures, so the stateful store is not installed globally.
+ beforeEach(() => {
+ installLiveStore(
+ {
+ read: mockCoreApi.readNamespacedSecret,
+ create: mockCoreApi.createNamespacedSecret,
+ replace: mockCoreApi.replaceNamespacedSecret,
+ },
+ {}
+ )
})
- mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})
- await (
- reconciler as unknown as {
- ensureDeployment: (w: unknown, r: unknown, l: string, s: unknown) => Promise<void>
+ const recipeWith = (res: SecretRes) =>
+ makeRecipe({
+ spec: {
+ workloads: [
+ { id: 'app', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 },
+ ],
+ resources: [res],
+ },
+ })
+ const ensureSecret = (res: SecretRes, recipe: WorkflowRecipeCRD) =>
+ (
+ reconciler as unknown as {
+ ensureSecret: (r: unknown, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+ ).ensureSecret(res, recipe)
+ const clearSecretCalls = () => {
+ for (const fn of [
+ mockCoreApi.readNamespacedSecret,
+ mockCoreApi.createNamespacedSecret,
+ mockCoreApi.replaceNamespacedSecret,
+ ]) {
+ fn.mockClear()
+ }
+ }
+ const onlyPost = (): CreateArgs => {
+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)
+ return mockCoreApi.createNamespacedSecret.mock.calls[0][0] as CreateArgs
+ }
+ const readLive = (name: string, namespace: string): Promise<SecretManifest> =>
+ mockCoreApi.readNamespacedSecret({ name, namespace }) as Promise<SecretManifest>
+ // Models an external writer relabelling the live Secret to another recipe.
+ const relabelToForeignOwner = async (name: string, namespace: string) => {
+ const live = await readLive(name, namespace)
+ await mockCoreApi.replaceNamespacedSecret({
+ name,
+ namespace,
+ body: {
+ ...live,
+ metadata: {
+ ...live.metadata,
+ labels: {
+ ...(live.metadata.labels as Record<string, string>),
+ 'clerum.io/recipe': 'other-recipe',
+ },
+ },
+ },
+ })
+ }
+
+ it('sends neither POST nor PUT for an existing generated-key Secret and keeps its keys', async () => {
+ const res: SecretRes = { id: 'creds', type: 'secret', generateKeys: ['password'] }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ const before = await readLive(name, created.namespace)
+ clearSecretCalls()
+
+ await ensureSecret(res, recipe)
+
+ // Witness: the gate read the live Secret before deciding not to write.
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)
+ const after = await readLive(name, created.namespace)
+ expect(after.data).toEqual(before.data)
+ })
+
+ it('refuses a generated-key Secret owned by another recipe without writing it', async () => {
+ const res: SecretRes = { id: 'creds', type: 'secret', generateKeys: ['password'] }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ await relabelToForeignOwner(name, created.namespace)
+ clearSecretCalls()
+
+ await expect(ensureSecret(res, recipe)).rejects.toThrow(
+ /not owned by WorkflowRecipe "test-recipe".*clerum\.io\/recipe=other-recipe/
+ )
+
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)
+ })
+
+ it('sends neither POST nor PUT for an unchanged user-data Secret', async () => {
+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ clearSecretCalls()
+
+ await ensureSecret(res, recipe)
+
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)
+ })
+
+ it('replaces a user-data Secret whose data changed, carrying the live resourceVersion', async () => {
+ const first: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }
+ await ensureSecret(first, recipeWith(first))
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ clearSecretCalls()
+
+ const second: SecretRes = { id: 'api', type: 'secret', data: { token: 'v2' } }
+ await ensureSecret(second, recipeWith(second))
+
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(1)
+ const replaced = mockCoreApi.replaceNamespacedSecret.mock.calls[0][0] as {
+ body: SecretManifest & { metadata: { resourceVersion?: string } }
}
- ).ensureDeployment(workload, recipe, 'minimal', {})
+ expect(replaced.body.data).toEqual({ token: Buffer.from('v2').toString('base64') })
+ expect(replaced.body.metadata.resourceVersion).toBe('1')
+ })
- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ it('refuses a user-data Secret owned by another recipe without writing it', async () => {
+ const res: SecretRes = { id: 'api', type: 'secret', data: { token: 'v1' } }
+ const recipe = recipeWith(res)
+ await ensureSecret(res, recipe)
+ const created = onlyPost()
+ const name = created.body.metadata.name
+ await relabelToForeignOwner(name, created.namespace)
+ clearSecretCalls()
+
+ await expect(ensureSecret(res, recipe)).rejects.toThrow(
+ /not owned by WorkflowRecipe "test-recipe".*clerum\.io\/recipe=other-recipe/
+ )
+
+ expect(readsOf(mockCoreApi.readNamespacedSecret, name, created.namespace)).toBe(1)
+ expect(postsOf(mockCoreApi.createNamespacedSecret, name)).toBe(0)
+ expect(putsOf(mockCoreApi.replaceNamespacedSecret, name)).toBe(0)
+ })
})
it('does NOT replace a StatefulSet whose spec-hash is unchanged', async () => {
@@ -2850,11 +3437,12 @@ describe('WorkflowRecipeReconciler', () => {
).ensureCronJob(workload, recipe, 'minimal', {})
expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()
- mockBatchApi.createNamespacedCronJob.mockRejectedValue({ code: 409 })
- mockBatchApi.readNamespacedCronJob.mockResolvedValue({
+ // Second apply: the read-first gate finds the object with the SAME hash.
+ mockBatchApi.readNamespacedCronJob.mockClear().mockResolvedValue({
metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },
spec: createdBody!.spec,
})
+ mockBatchApi.createNamespacedCronJob.mockClear()
mockBatchApi.replaceNamespacedCronJob.mockClear()
await (
@@ -2863,7 +3451,10 @@ describe('WorkflowRecipeReconciler', () => {
}
).ensureCronJob(workload, recipe, 'minimal', {})
- expect(mockBatchApi.replaceNamespacedCronJob).not.toHaveBeenCalled()
+ // Witness: the gate read the live object before deciding not to write.
+ expect(mockBatchApi.readNamespacedCronJob).toHaveBeenCalledTimes(1)
+ expect(mockBatchApi.createNamespacedCronJob).toHaveBeenCalledTimes(0)
+ expect(mockBatchApi.replaceNamespacedCronJob).toHaveBeenCalledTimes(0)
})
it('does NOT replace a Job whose spec-hash is unchanged', async () => {
@@ -2883,11 +3474,12 @@ describe('WorkflowRecipeReconciler', () => {
).ensureJob(workload, recipe, 'minimal', {})
expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()
- mockBatchApi.createNamespacedJob.mockRejectedValue({ code: 409 })
- mockBatchApi.readNamespacedJob.mockResolvedValue({
+ // Second apply: the read-first gate finds the object with the SAME hash.
+ mockBatchApi.readNamespacedJob.mockClear().mockResolvedValue({
metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },
spec: createdBody!.spec,
})
+ mockBatchApi.createNamespacedJob.mockClear()
mockBatchApi.replaceNamespacedJob.mockClear()
await (
@@ -2896,7 +3488,10 @@ describe('WorkflowRecipeReconciler', () => {
}
).ensureJob(workload, recipe, 'minimal', {})
- expect(mockBatchApi.replaceNamespacedJob).not.toHaveBeenCalled()
+ // Witness: the gate read the live object before deciding not to write.
+ expect(mockBatchApi.readNamespacedJob).toHaveBeenCalledTimes(1)
+ expect(mockBatchApi.createNamespacedJob).toHaveBeenCalledTimes(0)
+ expect(mockBatchApi.replaceNamespacedJob).toHaveBeenCalledTimes(0)
})
it('does NOT replace a DaemonSet whose spec-hash is unchanged', async () => {
@@ -2916,11 +3511,12 @@ describe('WorkflowRecipeReconciler', () => {
).ensureDaemonSet(workload, recipe, 'minimal', {})
expect(createdBody!.metadata.annotations[SPEC_HASH]).toBeDefined()
- mockAppsApi.createNamespacedDaemonSet.mockRejectedValue({ code: 409 })
- mockAppsApi.readNamespacedDaemonSet.mockResolvedValue({
+ // Second apply: the read-first gate finds the object with the SAME hash.
+ mockAppsApi.readNamespacedDaemonSet.mockClear().mockResolvedValue({
metadata: { resourceVersion: '9', annotations: createdBody!.metadata.annotations },
spec: createdBody!.spec,
})
+ mockAppsApi.createNamespacedDaemonSet.mockClear()
mockAppsApi.replaceNamespacedDaemonSet.mockClear()
await (
@@ -2929,19 +3525,24 @@ describe('WorkflowRecipeReconciler', () => {
}
).ensureDaemonSet(workload, recipe, 'minimal', {})
- expect(mockAppsApi.replaceNamespacedDaemonSet).not.toHaveBeenCalled()
+ // Witness: the gate read the live object before deciding not to write.
+ expect(mockAppsApi.readNamespacedDaemonSet).toHaveBeenCalledTimes(1)
+ expect(mockAppsApi.createNamespacedDaemonSet).toHaveBeenCalledTimes(0)
+ expect(mockAppsApi.replaceNamespacedDaemonSet).toHaveBeenCalledTimes(0)
})
- it('falls open and replaces when the existing object cannot be read (never silently skips)', async () => {
+ it('falls open and replaces when a transport failure hides the existing object (never silently skips)', async () => {
const recipe = makeRecipe()
const workload = recipe.spec.workloads![0]
- mockAppsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
- // First read (idempotency check) fails; the replace path's own read then succeeds.
+ // First read (the gate) times out at the socket; the replace path's own read then succeeds.
mockAppsApi.readNamespacedDeployment
.mockReset()
- .mockRejectedValueOnce(new Error('transient read timeout'))
+ .mockRejectedValueOnce(
+ Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), { code: 'ETIMEDOUT' })
+ )
.mockResolvedValue({ metadata: { resourceVersion: '9' } })
+ mockAppsApi.createNamespacedDeployment.mockClear()
mockAppsApi.replaceNamespacedDeployment.mockClear().mockResolvedValue({})
await (
@@ -2950,7 +3551,12 @@ describe('WorkflowRecipeReconciler', () => {
}
).ensureDeployment(workload, recipe, 'minimal', {})
+ expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(
+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion
+ ).toBe('9')
})
})
@@ -3712,10 +4318,22 @@ describe('WorkflowRecipeReconciler', () => {
})
it('uses create-or-replace pattern on 409 (3.11c)', async () => {
- mockAppsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ // Read-first race: the Deployment is absent at the gate read, another writer
+ // creates it before our POST (409), and the gate is re-entered on the live object.
+ mockAppsApi.readNamespacedDeployment
+ .mockReset()
+ .mockRejectedValueOnce({ code: 404 })
+ .mockResolvedValue({ metadata: { resourceVersion: '7' }, status: { readyReplicas: 1 } })
+ mockAppsApi.createNamespacedDeployment.mockReset().mockRejectedValueOnce({ code: 409 })
+ mockAppsApi.replaceNamespacedDeployment.mockReset().mockResolvedValue({})
+
await reconciler.reconcile(makeRecipe())
- expect(mockAppsApi.readNamespacedDeployment).toHaveBeenCalled()
- expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalled()
+
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(
+ mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0].body.metadata.resourceVersion
+ ).toBe('7')
})
// ─── Resource Creation ──────────────────────────────────────────
@@ -4986,8 +5604,9 @@ describe('WorkflowRecipeReconciler', () => {
})
it('handles partial failure (some workloads fail) (3.15c)', async () => {
+ const storeCreate = mockAppsApi.createNamespacedDeployment.getMockImplementation()!
mockAppsApi.createNamespacedDeployment
- .mockResolvedValueOnce({})
+ .mockImplementationOnce(storeCreate)
.mockRejectedValueOnce(new Error('quota exceeded'))
const recipe = makeRecipe({
@@ -5166,10 +5785,14 @@ describe('WorkflowRecipeReconciler', () => {
status: { readyReplicas: name === 'db' ? 1 : 0 },
})
)
- mockCoreApi.readNamespacedService.mockResolvedValue({
- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },
- spec: { clusterIP: '10.0.0.1' },
- })
+ // The legacy raw Services are live and owned by this recipe.
+ for (const legacy of ['db', 'db-headless']) {
+ await mockCoreApi.createNamespacedService({
+ namespace: 'sandbox-recipes',
+ body: { metadata: { name: legacy, labels: { 'clerum.io/recipe': 'existing-db' } } },
+ })
+ }
+ mockCoreApi.createNamespacedService.mockClear()
const firstResult = await reconciler.reconcile(recipe)
@@ -5199,10 +5822,6 @@ describe('WorkflowRecipeReconciler', () => {
spec: { replicas: 1 },
status: { readyReplicas: 1 },
})
- mockCoreApi.readNamespacedService.mockResolvedValue({
- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'existing-db' } },
- spec: { clusterIP: '10.0.0.1' },
- })
await reconciler.reconcile(recipe)
@@ -5282,8 +5901,9 @@ describe('WorkflowRecipeReconciler', () => {
// Scoped workload not ready yet → tearing down the raw "api" must be deferred.
// The raw Deployment carries this recipe's ownership label so cleanup recognizes
- // it as its own (issue #571 F2).
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
+ // it as its own (issue #571 F2). The scoped Deployment does not exist until
+ // this reconcile creates it, so its read-first gate read is a 404.
+ const notReady = {
metadata: {
resourceVersion: '1',
generation: 1,
@@ -5291,7 +5911,18 @@ describe('WorkflowRecipeReconciler', () => {
},
spec: { replicas: 1 },
status: { observedGeneration: 1, updatedReplicas: 1, readyReplicas: 0, availableReplicas: 0 },
- })
+ }
+ let scopedCreated = false
+ mockAppsApi.readNamespacedDeployment.mockImplementation(async ({ name }: { name: string }) => {
+ if (name === scoped && !scopedCreated) throw { code: 404 }
+ return structuredClone(notReady)
+ })
+ mockAppsApi.createNamespacedDeployment.mockImplementation(
+ async ({ body }: { body: { metadata: { name: string } } }) => {
+ if (body.metadata.name === scoped) scopedCreated = true
+ return {}
+ }
+ )
const firstResult = await reconciler.reconcile(recipe)
@@ -5388,12 +6019,14 @@ describe('WorkflowRecipeReconciler', () => {
await reconciler.reconcile(recipe)
- // PVC-mounting Deployment keeps the raw name (adopted), so the scoped Deployment
- // is never created and no cleanup/deletion is attempted.
- const createdNames = mockAppsApi.createNamespacedDeployment.mock.calls.map(
- ([arg]) => arg.body.metadata.name
+ // PVC-mounting Deployment keeps the raw name (adopted): the live raw object is
+ // replaced in place, the scoped Deployment is never created, and no
+ // cleanup/deletion is attempted.
+ const replacedNames = mockAppsApi.replaceNamespacedDeployment.mock.calls.map(
+ ([arg]) => arg.name
)
- expect(createdNames).toContain('api')
+ expect(replacedNames).toEqual(['api'])
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
expect(mockAppsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
})
@@ -7196,6 +7829,59 @@ describe('WorkflowRecipeReconciler', () => {
}
})
+ it('replaces a live ui egress policy whose set changed without a create probe', async () => {
+ const policyName = 'ui-egress-test-recipe'
+ let address = '93.184.216.10'
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: async () => ({ kind: 'ok', ipv4: [address], ipv6: [], ttlSeconds: 300 }),
+ })
+ const recipe = uiRecipeWithExternals([{ fqdn: 'api.stripe.com', port: 443 }])
+ await rec.reconcile(recipe)
+ const created = createdPolicy(policyName)
+ expect(created).toBeDefined()
+ // Stateful live policy: the first render, then whatever each PUT writes.
+ let live: k8s.V1NetworkPolicy = {
+ ...structuredClone(created!),
+ metadata: { ...created!.metadata, uid: 'uid-ui-egress-test-recipe', resourceVersion: '1' },
+ }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) =>
+ name === policyName ? Promise.resolve(structuredClone(live)) : Promise.reject({ code: 404 })
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ ({ name, body }: { name: string; body: k8s.V1NetworkPolicy }) => {
+ if (name !== policyName) return Promise.resolve({})
+ live = {
+ ...structuredClone(body),
+ metadata: {
+ ...body.metadata,
+ uid: 'uid-ui-egress-test-recipe',
+ resourceVersion: String(Number(live.metadata!.resourceVersion) + 1),
+ },
+ }
+ return Promise.resolve(structuredClone(live))
+ }
+ )
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ address = '93.184.216.20'
+
+ const result = await rec.reconcile(recipe)
+
+ expect(result.phase).toBe('active')
+ // Witness: the caller read the live policy it now hands to the apply.
+ expect(
+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, policyName, 'sandbox-ui')
+ ).toBeGreaterThanOrEqual(1)
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, policyName)).toBe(0)
+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, policyName)).toBe(1)
+ const cidrs = (live.spec?.egress ?? []).flatMap(rule =>
+ (rule.to ?? []).map(peer => peer.ipBlock?.cidr)
+ )
+ expect(cidrs).toContain('93.184.216.20/32')
+ })
+
// H-E (audit): a rename of the external FQDN onto the SAME resolved IP/port
// renders identical spec.egress, so the egress-signature gate alone would no-op
// and discard the re-attributed state annotation. acc.changed must force the
@@ -8142,6 +8828,56 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.requeueFixedInterval).toBe(false)
})
+ // The fixed interval exists for `deploying`, where the controller polls for a
+ // Pod it cannot watch and must beat the 240s mcp-host readiness deadline.
+ // Plugin Workload SDK policy-pending waits for an operator grant instead, which
+ // polling cannot advance, so it must requeue on the backoff path.
+ it('requeues a workflow awaiting Plugin Workload SDK policy without a fixed interval', async () => {
+ const workflowReconcile = vi.fn().mockResolvedValue({
+ phase: 'active',
+ message: 'Workflow running',
+ workflowPhase: 'running',
+ pluginWorkloadSdkPolicyPending: true,
+ })
+ ;(
+ reconciler as unknown as {
+ config: { pluginWorkloadSdkEnabled: boolean }
+ }
+ ).config.pluginWorkloadSdkEnabled = true
+ ;(
+ reconciler as unknown as {
+ workflowReconciler: {
+ reconcile: typeof workflowReconcile
+ validateWorkflowSpec: () => undefined
+ }
+ }
+ ).workflowReconciler = { reconcile: workflowReconcile, validateWorkflowSpec: () => undefined }
+
+ const recipe = makeRecipe({
+ spec: {
+ agent: { provider: 'zai', model: 'glm-4.7' },
+ steps: [{ id: 'research', instruction: 'run' }],
+ pluginWorkloadSdk: { promptBridge: {}, allowedCallers: ['research'] },
+ },
+ status: { phase: 'candidate' },
+ })
+
+ const result = await reconciler.reconcile(recipe)
+
+ // Liveness witness: the workflow lane ran and carried policy-pending into a
+ // result whose projection publishes PluginWorkloadSdkPolicyPending=True.
+ expect(workflowReconcile).toHaveBeenCalledTimes(1)
+ expect(result.pluginWorkloadSdkPolicyPending).toBe(true)
+ expect(reconciler.projectPluginWorkloadSdk(recipe, result).conditions).toContainEqual(
+ expect.objectContaining({
+ type: PLUGIN_WORKLOAD_SDK_POLICY_PENDING_CONDITION_TYPE,
+ status: 'True',
+ })
+ )
+ expect(result.requeueAfterMs).toBe(WORKFLOW_PROGRESS_REQUEUE_BASE_MS)
+ expect(result.requeueFixedInterval).not.toBe(true)
+ })
+
it('degrades (retryable) workflow transport workloads when child McpServer pre-deploy fails', async () => {
const workflowReconcile = vi.fn().mockResolvedValue({
phase: 'deploying',
@@ -10613,8 +11349,21 @@ describe('WorkflowRecipeReconciler', () => {
})
}
+ // The apply reads first, so a test about creating the policy must start
+ // with it absent; the global default models a converged live policy.
+ function coordinatorGfsPolicyAbsent(): void {
+ const base = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ (args: { name: string; namespace: string }) =>
+ args.name === 'gfs-policy-workflow-coordinator-to-gfs'
+ ? Promise.reject({ code: 404 })
+ : base(args)
+ )
+ }
+
it('creates the exact coordinator GFS policy when publishTargets become present', async () => {
stubSteadyWorkflowRuntime()
+ coordinatorGfsPolicyAbsent()
const recipe = activeWorkflow([{ drive: 'main', target: 'published-results' }])
const result = await reconciler.reconcile(recipe)
@@ -10903,6 +11652,7 @@ describe('WorkflowRecipeReconciler', () => {
},
])('converges the present policy before returning from $branch', async testCase => {
stubSteadyWorkflowRuntime()
+ coordinatorGfsPolicyAbsent()
const workflowRuntime = (
reconciler as unknown as { workflowReconciler: Record<string, unknown> }
).workflowReconciler
@@ -11014,6 +11764,125 @@ describe('WorkflowRecipeReconciler', () => {
expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
+ describe('read-first apply of the coordinator GFS policy', () => {
+ const gfsPolicyName = 'gfs-policy-workflow-coordinator-to-gfs'
+ const publishTargets = [{ drive: 'main', target: 'published-results' }]
+
+ // Stateful double for the coordinator GFS policy only; every other
+ // NetworkPolicy keeps the behaviour installed by the global beforeEach.
+ const installGfsPolicyStore = (seed?: k8s.V1NetworkPolicy) => {
+ let live = seed ? structuredClone(seed) : undefined
+ const baseRead = mockNetworkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
+ const baseCreate = mockNetworkingApi.createNamespacedNetworkPolicy.getMockImplementation()!
+ const baseReplace =
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.getMockImplementation()!
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ async (args: { name: string; namespace: string }) => {
+ if (args.name !== gfsPolicyName) return baseRead(args)
+ if (!live) throw { code: 404 }
+ return structuredClone(live)
+ }
+ )
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockImplementation(
+ async (args: { namespace: string; body: k8s.V1NetworkPolicy }) => {
+ if (args.body.metadata?.name !== gfsPolicyName) return baseCreate(args)
+ if (live) throw { code: 409 }
+ live = {
+ ...structuredClone(args.body),
+ metadata: { ...args.body.metadata, uid: 'uid-owned-policy', resourceVersion: '1' },
+ }
+ return structuredClone(live)
+ }
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockImplementation(
+ async (args: { name: string; namespace: string; body: k8s.V1NetworkPolicy }) => {
+ if (args.name !== gfsPolicyName) return baseReplace(args)
+ if (args.body.metadata?.resourceVersion !== live?.metadata?.resourceVersion) {
+ throw { code: 409 }
+ }
+ live = {
+ ...structuredClone(args.body),
+ metadata: {
+ ...args.body.metadata,
+ uid: 'uid-owned-policy',
+ resourceVersion: String(Number(live!.metadata!.resourceVersion) + 1),
+ },
+ }
+ return structuredClone(live)
+ }
+ )
+ return { live: () => live }
+ }
+ const clearPolicyCalls = () => {
+ for (const fn of [
+ mockNetworkingApi.readNamespacedNetworkPolicy,
+ mockNetworkingApi.createNamespacedNetworkPolicy,
+ mockNetworkingApi.replaceNamespacedNetworkPolicy,
+ ]) {
+ fn.mockClear()
+ }
+ }
+
+ it('sends neither POST nor PUT for a converged policy on the next pass', async () => {
+ stubSteadyWorkflowRuntime()
+ installGfsPolicyStore()
+ await reconciler.reconcile(activeWorkflow(publishTargets))
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(1)
+ clearPolicyCalls()
+
+ await reconciler.reconcile(activeWorkflow(publishTargets))
+
+ // Witness: the gate read the live policy before deciding not to write.
+ expect(
+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')
+ ).toBe(1)
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ })
+
+ it('replaces a live owned policy whose spec drifted, without a create probe', async () => {
+ stubSteadyWorkflowRuntime()
+ const drifted = buildCoordinatorGfsNetworkPolicy({
+ recipeName: 'gfs-policy-workflow',
+ sandboxNamespace: 'sandbox-recipes',
+ })
+ drifted.metadata = { ...drifted.metadata, uid: 'uid-owned-policy', resourceVersion: '7' }
+ drifted.spec!.egress![0].ports = [{ port: 9999, protocol: 'TCP' }]
+ const store = installGfsPolicyStore(drifted)
+ clearPolicyCalls()
+
+ await reconciler.reconcile(activeWorkflow(publishTargets))
+
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(1)
+ expect(store.live()?.spec?.egress?.[0].ports).toEqual([{ port: 8087, protocol: 'TCP' }])
+ expect(store.live()?.metadata?.resourceVersion).toBe('8')
+ })
+
+ it('refuses a homonymous policy owned elsewhere without writing it', async () => {
+ stubSteadyWorkflowRuntime()
+ installGfsPolicyStore({
+ metadata: {
+ name: gfsPolicyName,
+ uid: 'uid-foreign-policy',
+ resourceVersion: '7',
+ labels: { 'clerum.io/managed-by': 'operator', 'clerum.io/recipe': 'other-recipe' },
+ },
+ })
+ clearPolicyCalls()
+
+ await expect(reconciler.reconcile(activeWorkflow(publishTargets))).rejects.toThrow(
+ 'existing policy is not owned by WRC'
+ )
+
+ expect(
+ readsOf(mockNetworkingApi.readNamespacedNetworkPolicy, gfsPolicyName, 'sandbox-recipes')
+ ).toBe(1)
+ expect(postsOf(mockNetworkingApi.createNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ expect(putsOf(mockNetworkingApi.replaceNamespacedNetworkPolicy, gfsPolicyName)).toBe(0)
+ })
+ })
+
it('refuses to delete a homonymous policy not owned by the exact recipe', async () => {
stubSteadyWorkflowRuntime()
mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValueOnce({
@@ -11065,6 +11934,7 @@ describe('WorkflowRecipeReconciler', () => {
})
it('opens coordinator GFS egress once only after a successful first deploy', async () => {
+ coordinatorGfsPolicyAbsent()
const workflowReconcile = vi.fn().mockResolvedValue({
phase: 'deploying',
message: 'Workflow infrastructure created',
@@ -11691,8 +12561,18 @@ describe('WorkflowRecipeReconciler', () => {
// ─── resolveWorkloadResourceName: prefixed names for workflow recipes ──
- it('uses the persisted scoped workload instance on 409 update for catalog recipes', async () => {
- mockAppsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ it('uses the persisted scoped workload instance on update for catalog recipes', async () => {
+ // The scoped Deployment already exists with another spec-hash, so the
+ // read-first gate replaces it: both the read and the PUT must target the
+ // scoped name, never the bare workload id. Two reads: the gate, then the
+ // replace re-reads the live resourceVersion.
+ const liveScoped = {
+ metadata: { resourceVersion: '4', annotations: { 'clerum.io/spec-hash': 'stale' } },
+ spec: {},
+ }
+ mockAppsApi.readNamespacedDeployment
+ .mockResolvedValueOnce(liveScoped)
+ .mockResolvedValueOnce(liveScoped)
const recipe = makeRecipe({
metadata: { name: 'simple-recipe', namespace: 'sandbox-recipes', uid: 'uid-nw' },
@@ -11709,6 +12589,11 @@ describe('WorkflowRecipeReconciler', () => {
const readCall = mockAppsApi.readNamespacedDeployment.mock.calls[0][0]
expect(readCall.name).toBe(expectedName)
expect(readCall.name).not.toBe('web-server')
+ expect(mockAppsApi.createNamespacedDeployment).toHaveBeenCalledTimes(0)
+ expect(mockAppsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ const replaced = mockAppsApi.replaceNamespacedDeployment.mock.calls[0][0]
+ expect(replaced.name).toBe(expectedName)
+ expect(replaced.body.metadata.resourceVersion).toBe('4')
})
it('patchStatus stores terminal workflowExecution phase for pre-runtime workflow failures', async () => {
@@ -12060,11 +12945,14 @@ describe('WorkflowRecipeReconciler', () => {
data: { 'signing-secret': 'YmFzZTY0LXNlY3JldA==' },
})
}
- mockAppsApi.readNamespacedDeployment.mockReset()
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: overrides?.deploymentReady === false ? 0 : 1 },
- })
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedDeployment,
+ create: mockAppsApi.createNamespacedDeployment,
+ replace: mockAppsApi.replaceNamespacedDeployment,
+ },
+ { readyReplicas: overrides?.deploymentReady === false ? 0 : 1 }
+ )
return recipe
}
@@ -12102,6 +12990,72 @@ describe('WorkflowRecipeReconciler', () => {
expect(allFalse).toBe(true)
})
+ describe('read-first apply of the gateway objects', () => {
+ const ns = 'sandbox-recipes'
+ const gateway = 'wf-wh-recipe-webhook-gateway'
+ const gatewayConfig = 'wf-wh-recipe-webhook-gateway-config'
+
+ // Two reconciles of the same recipe: the second must find every gateway
+ // object live with the desired spec-hash and write none of them.
+ const reconcileTwice = async (
+ api: {
+ read: ReturnType<typeof vi.fn>
+ create: ReturnType<typeof vi.fn>
+ replace: ReturnType<typeof vi.fn>
+ },
+ name: string
+ ) => {
+ const recipe = recipeWithWebhook()
+ await reconciler.reconcile(recipe)
+ expect(postsOf(api.create, name)).toBe(1)
+ api.read.mockClear()
+ api.create.mockClear()
+ api.replace.mockClear()
+ await reconciler.reconcile(recipe)
+ }
+
+ it('sends neither POST nor PUT for the unchanged gateway ConfigMap', async () => {
+ const api = {
+ read: mockCoreApi.readNamespacedConfigMap,
+ create: mockCoreApi.createNamespacedConfigMap,
+ replace: mockCoreApi.replaceNamespacedConfigMap,
+ }
+ await reconcileTwice(api, gatewayConfig)
+
+ expect(readsOf(api.read, gatewayConfig, ns)).toBe(1)
+ expect(postsOf(api.create, gatewayConfig)).toBe(0)
+ expect(putsOf(api.replace, gatewayConfig)).toBe(0)
+ })
+
+ it('sends neither POST nor PUT for the unchanged gateway Deployment', async () => {
+ const api = {
+ read: mockAppsApi.readNamespacedDeployment,
+ create: mockAppsApi.createNamespacedDeployment,
+ replace: mockAppsApi.replaceNamespacedDeployment,
+ }
+ await reconcileTwice(api, gateway)
+
+ // The readiness check reads the gateway Deployment too; the gate read
+ // is the one that precedes it.
+ expect(readsOf(api.read, gateway, ns)).toBe(2)
+ expect(postsOf(api.create, gateway)).toBe(0)
+ expect(putsOf(api.replace, gateway)).toBe(0)
+ })
+
+ it('sends neither POST nor PUT for the unchanged gateway Service', async () => {
+ const api = {
+ read: mockCoreApi.readNamespacedService,
+ create: mockCoreApi.createNamespacedService,
+ replace: mockCoreApi.replaceNamespacedService,
+ }
+ await reconcileTwice(api, gateway)
+
+ expect(readsOf(api.read, gateway, ns)).toBe(1)
+ expect(postsOf(api.create, gateway)).toBe(0)
+ expect(putsOf(api.replace, gateway)).toBe(0)
+ })
+ })
+
it('T6 (#575) converges the 3 gateway NetworkPolicies through live reads', async () => {
mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
@@ -12336,11 +13290,14 @@ describe('WorkflowRecipeReconciler', () => {
data: { 'signing-secret': 'YmFzZTY0LXNlY3JldA==' },
})
}
- mockAppsApi.readNamespacedDeployment.mockReset()
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 1 },
- })
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedDeployment,
+ create: mockAppsApi.createNamespacedDeployment,
+ replace: mockAppsApi.replaceNamespacedDeployment,
+ },
+ { readyReplicas: 1 }
+ )
return recipe
}
@@ -13161,10 +14118,14 @@ describe('WorkflowRecipeReconciler', () => {
})
it('re-reconciles a recipe failed by shared mcp-server internal dependency boundary', async () => {
- mockAppsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 1 },
- })
+ installLiveStore(
+ {
+ read: mockAppsApi.readNamespacedDeployment,
+ create: mockAppsApi.createNamespacedDeployment,
+ replace: mockAppsApi.replaceNamespacedDeployment,
+ },
+ { readyReplicas: 1 }
+ )
const recipe = makeRecipe({
spec: {
workloads: [
diff --git a/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts b/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts
index 5580e5ca9..bbd7f74a2 100644
--- a/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts
+++ b/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts
@@ -108,7 +108,8 @@ describe('UI egress dedicated writer metadata and lifecycle', () => {
await fixture.reconcile()
expect(fixture.read).toHaveBeenCalledTimes(2)
- expect(fixture.create).toHaveBeenCalledTimes(1)
+ // The policy already exists, so the read-first apply replaces it without a create probe.
+ expect(fixture.create).not.toHaveBeenCalled()
expect(fixture.replace).toHaveBeenCalledTimes(1)
expect(fixture.live().metadata).toMatchObject({
labels: {
@@ -173,7 +174,7 @@ describe('UI egress dedicated writer metadata and lifecycle', () => {
),
})
expect(fixture.read).toHaveBeenCalledTimes(2)
- expect(fixture.create).toHaveBeenCalledTimes(1)
+ expect(fixture.create).not.toHaveBeenCalled()
expect(fixture.replace).not.toHaveBeenCalled()
}
)
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index f4e9afc6a..4b3c90294 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1104,7 +1104,8 @@ export class WorkflowRecipeReconciler {
const result = await waitForExternalEgressReady(
this.delegationDeps,
externalServers,
- namespace
+ namespace,
+ recipe.metadata.name
)
if (!result.ready) {
throw externalEgressReadinessError(recipe.metadata.name, result)
@@ -1116,7 +1117,8 @@ export class WorkflowRecipeReconciler {
const { ready, pending } = await waitForNetworkReady(
this.delegationDeps,
genericServers,
- namespace
+ namespace,
+ recipe.metadata.name
)
if (!ready) {
createLogger('wrc', recipe.metadata.name).warn('Generic network readiness not confirmed', {
@@ -2260,14 +2262,17 @@ export class WorkflowRecipeReconciler {
// degrade the poll cadence toward the 60s cap and re-expose the 240s
// mcp-host readiness deadline this requeue exists to beat (see §5 and
// requeueFixedInterval doc).
+ //
+ // Plugin Workload SDK policy-pending also requeues, but on the backoff
+ // path: it waits for an operator grant, which arrives by event or by the
+ // 30s credential-refresh floor. Polling cannot advance it, and it has no
+ // deadline that bounds a fixed-interval loop.
requeueAfterMs: result.skipStatusPatch
? TRANSIENT_REQUEUE_BASE_MS
: result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending
? WORKFLOW_PROGRESS_REQUEUE_BASE_MS
: undefined,
- requeueFixedInterval:
- !result.skipStatusPatch &&
- (result.phase === 'deploying' || result.pluginWorkloadSdkPolicyPending === true),
+ requeueFixedInterval: !result.skipStatusPatch && result.phase === 'deploying',
internalDependencyConditions: workflowInternalDependencyConditions,
secretOwnershipConditions: workflowSecretOwnershipConditions,
}
@@ -2736,7 +2741,12 @@ export class WorkflowRecipeReconciler {
body: svc,
})
},
- `Service "${svcName}"`
+ `Service "${svcName}"`,
+ {
+ manifest: svc,
+ readExisting: () =>
+ this.coreApi.readNamespacedService({ name: svcName, namespace: svcNs }),
+ }
)
}
}
@@ -2938,8 +2948,9 @@ export class WorkflowRecipeReconciler {
sdkOnlyBootstrapPending
? TRANSIENT_REQUEUE_BASE_MS
: undefined,
- requeueFixedInterval:
- sdkOnlyPolicyPending || sdkOnlyRuntime?.phase === 'deploying' || sdkOnlyBootstrapPending,
+ // Policy-pending waits for an operator grant (event or the 30s refresh
+ // floor), so it requeues with backoff; only boot progress is fixed-interval.
+ requeueFixedInterval: sdkOnlyRuntime?.phase === 'deploying' || sdkOnlyBootstrapPending,
}
} catch (error) {
// Transient failures must not brick a healthy recipe at the terminal,
@@ -3286,7 +3297,15 @@ export class WorkflowRecipeReconciler {
body: built.configConfigMap,
})
},
- `WebhookGateway ConfigMap "${gatewayConfigMapName(recipeName)}"`
+ `WebhookGateway ConfigMap "${gatewayConfigMapName(recipeName)}"`,
+ {
+ manifest: built.configConfigMap,
+ readExisting: () =>
+ this.coreApi.readNamespacedConfigMap({
+ name: gatewayConfigMapName(recipeName),
+ namespace: ns,
+ }),
+ }
)
await this.createOrReplace(
@@ -3303,7 +3322,15 @@ export class WorkflowRecipeReconciler {
body: built.deployment,
})
},
- `WebhookGateway Deployment "${gatewayResourceName(recipeName)}"`
+ `WebhookGateway Deployment "${gatewayResourceName(recipeName)}"`,
+ {
+ manifest: built.deployment,
+ readExisting: () =>
+ this.appsApi.readNamespacedDeployment({
+ name: gatewayResourceName(recipeName),
+ namespace: ns,
+ }),
+ }
)
await this.createOrReplace(
@@ -3321,7 +3348,15 @@ export class WorkflowRecipeReconciler {
body: built.service,
})
},
- `WebhookGateway Service "${gatewayServiceName(recipeName)}"`
+ `WebhookGateway Service "${gatewayServiceName(recipeName)}"`,
+ {
+ manifest: built.service,
+ readExisting: () =>
+ this.coreApi.readNamespacedService({
+ name: gatewayServiceName(recipeName),
+ namespace: ns,
+ }),
+ }
)
// The gateway policies use the same live convergence path as other
@@ -3596,7 +3631,12 @@ export class WorkflowRecipeReconciler {
body: svc,
})
},
- `Service "${svcName}"`
+ `Service "${svcName}"`,
+ {
+ manifest: svc,
+ readExisting: () =>
+ this.coreApi.readNamespacedService({ name: svcName, namespace: svcNs }),
+ }
)
} catch (err) {
createLogger('wrc', recipe.metadata.name).error(
@@ -4023,15 +4063,18 @@ export class WorkflowRecipeReconciler {
/**
* Create-or-replace a managed object. When `idempotency` is supplied, the
- * desired manifest is stamped with a spec-hash annotation and, on the
- * already-exists (409) path, the existing object's stamped hash is compared to
- * the desired hash — if they match the replace PUT is SKIPPED entirely.
+ * desired manifest is stamped with a spec-hash annotation and the object is
+ * READ FIRST: absent → POST, present with the desired hash → no write at all,
+ * present with another hash → PUT. Without `idempotency` the object is
+ * created first and replaced on 409.
*
* This is what stops the generation churn: WRC reconciles every workload on a
* periodic resync, and an unconditional full replace re-defaults server-managed
* fields, bumping metadata.generation with no real change (→ a degraded↔active
* status flap and a downstream HCC NetworkPolicy no-op write storm). Skipping
- * unchanged writes makes the reconcile idempotent. See specHash.ts.
+ * unchanged writes makes the reconcile idempotent, and reading first keeps the
+ * steady state from sending a POST only to have it rejected with 409.
+ * See specHash.ts.
*/
private async createOrReplace(
createFn: () => Promise<unknown>,
@@ -4042,54 +4085,81 @@ export class WorkflowRecipeReconciler {
readExisting: () => Promise<{ metadata?: { annotations?: { [key: string]: string } } } | null>
}
): Promise<void> {
- if (idempotency) stampSpecHash(idempotency.manifest)
+ if (idempotency) {
+ stampSpecHash(idempotency.manifest)
+ const gate = await this.applyGate(idempotency)
+ if (gate === 'unchanged') {
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Resource spec hash unchanged; skipping update',
+ { label }
+ )
+ return
+ }
+ if (gate === 'changed') {
+ await this.replaceResource(replaceFn, label)
+ return
+ }
+ }
try {
await createFn()
createLogger('wrc', 'workflow-recipes').info('Created resource', { label })
} catch (error: unknown) {
- if (getErrorCode(error) === 409) {
- if (idempotency && (await this.applyIsNoop(idempotency))) {
- createLogger('wrc', 'workflow-recipes').info(
- 'Resource spec hash unchanged; skipping update',
- { label }
- )
- return
- }
- try {
- await replaceFn()
- createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })
- } catch (updateError) {
- createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {
- label,
- err: updateError,
- })
- throw updateError
- }
- } else {
+ if (getErrorCode(error) !== 409) {
createLogger('wrc', 'workflow-recipes').error('Failed to create resource', {
label,
err: error,
})
throw error
}
+ // Without a gate this is the create-first probe. With one, another writer
+ // created the object between our read and this POST: re-enter the gate.
+ if (idempotency && (await this.applyGate(idempotency)) === 'unchanged') {
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Resource spec hash unchanged; skipping update',
+ { label }
+ )
+ return
+ }
+ await this.replaceResource(replaceFn, label)
+ }
+ }
+
+ private async replaceResource(replaceFn: () => Promise<unknown>, label: string): Promise<void> {
+ try {
+ await replaceFn()
+ createLogger('wrc', 'workflow-recipes').info('Updated resource', { label })
+ } catch (updateError) {
+ createLogger('wrc', 'workflow-recipes').error('Failed to update resource', {
+ label,
+ err: updateError,
+ })
+ throw updateError
}
}
/**
- * True when the existing object already carries the desired spec-hash, so the
- * replace would be a no-op. A read failure returns false (fall through to
- * replace) — never skip an update we cannot prove is unnecessary.
+ * Read the live object and decide the write: `absent` (404) → create,
+ * `unchanged` (it carries the desired spec-hash) → no write, `changed` → replace.
+ *
+ * A read failure that carries an apiserver status or a transport signature
+ * returns `changed` — never skip an update we cannot prove is unnecessary.
+ * Any other failure is not a read outcome but a domain or programming error,
+ * and it propagates instead of turning into a silent overwrite.
*/
- private async applyIsNoop(idempotency: {
+ private async applyGate(idempotency: {
manifest: { metadata?: { annotations?: { [key: string]: string } } }
readExisting: () => Promise<{ metadata?: { annotations?: { [key: string]: string } } } | null>
- }): Promise<boolean> {
+ }): Promise<'absent' | 'unchanged' | 'changed'> {
+ let existing: { metadata?: { annotations?: { [key: string]: string } } } | null
try {
- const existing = await idempotency.readExisting()
- return specHashUnchanged(idempotency.manifest, existing ?? null)
- } catch {
- return false
+ existing = await idempotency.readExisting()
+ } catch (error) {
+ if (getErrorCode(error) === 404) return 'absent'
+ if (getErrorCode(error) !== undefined || isRetryableInfraError(error)) return 'changed'
+ throw error
}
+ if (existing == null) return 'absent'
+ return specHashUnchanged(idempotency.manifest, existing) ? 'unchanged' : 'changed'
}
private stableComparableString(value: unknown): string {
@@ -4617,7 +4687,7 @@ export class WorkflowRecipeReconciler {
ns,
})
} else {
- await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name)
+ await this.applyOwnedRecipeNetworkPolicy(policy, ns, recipe.metadata.name, existing)
// #299: the policy has landed — record the set it actually enforces.
this.logResolvedEgressSet(
recipe.metadata.name,
@@ -5416,51 +5486,65 @@ export class WorkflowRecipeReconciler {
return live
}
+ /**
+ * `existing` is the live policy the caller already read this pass (or null
+ * when there is none). When one exists the create probe is skipped and the
+ * replace runs directly; the replace still re-reads and fences on that fresh
+ * resourceVersion, because a contraction earlier in the same pass may have
+ * written the policy after the caller's snapshot.
+ */
private async applyOwnedRecipeNetworkPolicy(
policy: k8s.V1NetworkPolicy,
namespace: string,
- recipeName: string
+ recipeName: string,
+ existing: k8s.V1NetworkPolicy | null
): Promise<void> {
const policyName = policy.metadata?.name
if (!policyName) throw new Error('Recipe NetworkPolicy requires a name')
- await this.createOrReplace(
- () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),
- async () => {
- let live: k8s.V1NetworkPolicy
- try {
- live = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
- } catch (error: unknown) {
- if (getErrorCode(error) === 404) {
- throw new NetworkPolicyReplanRequiredError(
- `NetworkPolicy "${policyName}" in ${namespace} disappeared after create conflict; a fresh reconciliation is required`,
- { cause: error }
- )
- }
- throw error
- }
- this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)
- if (!live.metadata?.resourceVersion) {
- throw new NetworkPolicyOwnershipConflictError(
- `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`
+ const label = `NetworkPolicy "${policyName}" in ${namespace}`
+ const replaceLive = async () => {
+ let live: k8s.V1NetworkPolicy
+ try {
+ live = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ } catch (error: unknown) {
+ if (getErrorCode(error) === 404) {
+ throw new NetworkPolicyReplanRequiredError(
+ `NetworkPolicy "${policyName}" in ${namespace} disappeared before replace; a fresh reconciliation is required`,
+ { cause: error }
)
}
- const replacement = buildNetworkPolicyReplacement(policy, live)
- try {
- return await this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- body: replacement,
- })
- } catch (error: unknown) {
- // This PUT is fenced by the freshly checked owner/resourceVersion.
- // Losing that snapshot is unfinished convergence, not bad user intent.
- throwNetworkPolicyMutationError(error, policyName, namespace)
- }
- },
- `NetworkPolicy "${policyName}" in ${namespace}`
+ throw error
+ }
+ this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)
+ if (!live.metadata?.resourceVersion) {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`
+ )
+ }
+ const replacement = buildNetworkPolicyReplacement(policy, live)
+ try {
+ return await this.networkingApi.replaceNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ body: replacement,
+ })
+ } catch (error: unknown) {
+ // This PUT is fenced by the freshly checked owner/resourceVersion.
+ // Losing that snapshot is unfinished convergence, not bad user intent.
+ throwNetworkPolicyMutationError(error, policyName, namespace)
+ }
+ }
+ if (existing) {
+ await this.replaceResource(replaceLive, label)
+ return
+ }
+ await this.createOrReplace(
+ () => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),
+ replaceLive,
+ label
)
}
@@ -5879,30 +5963,60 @@ export class WorkflowRecipeReconciler {
const namespace = this.config.sandboxNamespace
const policy = this.coordinatorGfsNetworkPolicy(recipe)
const name = policy.metadata!.name!
+ const label = `NetworkPolicy "${name}" in ${namespace}`
+ const matchesDesired = (existing: k8s.V1NetworkPolicy): boolean => {
+ const desiredLabels = policy.metadata?.labels ?? {}
+ const existingLabels = existing.metadata?.labels ?? {}
+ const labelsMatch = Object.entries(desiredLabels).every(
+ ([key, value]) => existingLabels[key] === value
+ )
+ return labelsMatch && JSON.stringify(existing.spec) === JSON.stringify(policy.spec)
+ }
+ const replaceWith = (existing: k8s.V1NetworkPolicy) => {
+ policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
+ return this.networkingApi.replaceNamespacedNetworkPolicy({
+ name,
+ namespace,
+ body: policy,
+ })
+ }
+
+ // Read first so a converged policy costs one GET instead of a POST that
+ // the apiserver rejects with 409. Only a 404 falls through to the create
+ // path; any other read failure propagates.
+ let existing: k8s.V1NetworkPolicy | null
+ try {
+ existing = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
+ } catch (error: unknown) {
+ if (getErrorCode(error) !== 404) throw error
+ existing = null
+ }
+ if (existing) {
+ this.assertCoordinatorGfsNetworkPolicyOwnership(existing, recipe.metadata.name, namespace)
+ if (matchesDesired(existing)) {
+ createLogger('wrc', recipe.metadata.name).info(
+ 'NetworkPolicy matches desired state; skipping update',
+ { label }
+ )
+ return
+ }
+ const live = existing
+ await this.replaceResource(() => replaceWith(live), label)
+ return
+ }
+
await this.createOrReplace(
() => this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy }),
async () => {
- const existing = await this.networkingApi.readNamespacedNetworkPolicy({
- name,
- namespace,
- })
- this.assertCoordinatorGfsNetworkPolicyOwnership(existing, recipe.metadata.name, namespace)
- const desiredLabels = policy.metadata?.labels ?? {}
- const existingLabels = existing.metadata?.labels ?? {}
- const labelsMatch = Object.entries(desiredLabels).every(
- ([key, value]) => existingLabels[key] === value
- )
- if (labelsMatch && JSON.stringify(existing.spec) === JSON.stringify(policy.spec)) {
- return existing
- }
- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
- return this.networkingApi.replaceNamespacedNetworkPolicy({
+ const raced = await this.networkingApi.readNamespacedNetworkPolicy({
name,
namespace,
- body: policy,
})
+ this.assertCoordinatorGfsNetworkPolicyOwnership(raced, recipe.metadata.name, namespace)
+ if (matchesDesired(raced)) return raced
+ return replaceWith(raced)
},
- `NetworkPolicy "${name}" in ${namespace}`
+ label
)
}
@@ -7019,7 +7133,46 @@ export class WorkflowRecipeReconciler {
ns,
recipe.metadata.namespace ?? this.config.sandboxNamespace
)
- if (res.generateKeys && res.generateKeys.length > 0) {
+ const label = `Secret "${res.id}" in ${ns}`
+ const generatesKeys = Boolean(res.generateKeys && res.generateKeys.length > 0)
+
+ // Read first, compared structurally rather than by spec-hash: a hash of
+ // user data written to an annotation would expose it to offline guessing,
+ // and generated keys differ on every build so a hash would never match.
+ // Only a 404 falls through to the create path below; any other read
+ // failure propagates.
+ let existing: k8s.V1Secret | null
+ try {
+ existing = await this.coreApi.readNamespacedSecret({ name, namespace: ns })
+ } catch (error) {
+ if (getErrorCode(error) !== 404) throw error
+ existing = null
+ }
+ if (existing) {
+ this.assertExistingResourceOwnedByRecipe(existing, name, res.id, recipe, ns, 'Secret')
+ if (generatesKeys) {
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Secret with generated keys exists; keeping its keys',
+ { label }
+ )
+ return
+ }
+ if (this.secretMatchesDesired(manifest, existing)) {
+ createLogger('wrc', 'workflow-recipes').info(
+ 'Secret matches desired state; skipping update',
+ { label }
+ )
+ return
+ }
+ manifest.metadata!.resourceVersion = existing.metadata?.resourceVersion
+ await this.replaceResource(
+ () => this.coreApi.replaceNamespacedSecret({ name, namespace: ns, body: manifest }),
+ label
+ )
+ return
+ }
+
+ if (generatesKeys) {
// Create-only semantics: random credentials are generated once and must not be
// rotated on subsequent reconcile loops (rotation would silently break running
// workloads that have the original values mounted via envFrom/envSecret).
@@ -7032,7 +7185,7 @@ export class WorkflowRecipeReconciler {
const existing = await this.coreApi.readNamespacedSecret({ name, namespace: ns })
this.assertExistingResourceOwnedByRecipe(existing, name, res.id, recipe, ns, 'Secret')
},
- `Secret "${res.id}" in ${ns}`
+ label
)
} else {
await this.createOrReplace(
@@ -7049,11 +7202,34 @@ export class WorkflowRecipeReconciler {
body: manifest,
})
},
- `Secret "${res.id}" in ${ns}`
+ label
)
}
}
+ /**
+ * A live Secret needs no write when its data and type equal the desired ones,
+ * it carries every desired label, and its controller ownerReference points at
+ * the same owner uid (none on either side for a cross-namespace Secret).
+ */
+ private secretMatchesDesired(desired: k8s.V1Secret, existing: k8s.V1Secret): boolean {
+ if (
+ this.stableComparableString(desired.data ?? {}) !==
+ this.stableComparableString(existing.data ?? {})
+ ) {
+ return false
+ }
+ if (desired.type !== existing.type) return false
+ const liveLabels = existing.metadata?.labels ?? {}
+ const labelsMatch = Object.entries(desired.metadata?.labels ?? {}).every(
+ ([key, value]) => liveLabels[key] === value
+ )
+ if (!labelsMatch) return false
+ const controllerUid = (secret: k8s.V1Secret) =>
+ secret.metadata?.ownerReferences?.find(ref => ref.controller)?.uid
+ return controllerUid(desired) === controllerUid(existing)
+ }
+
/**
* Refuse to mutate a pre-existing Secret/ConfigMap that is not owned by this
* recipe (`clerum.io/recipe` label). Mirrors assertExistingResourcePvcOwnedByRecipe
@@ -7199,7 +7375,15 @@ export class WorkflowRecipeReconciler {
body: manifest,
})
},
- `ConfigMap "${res.id}" in ${ns}`
+ `ConfigMap "${res.id}" in ${ns}`,
+ // An unchanged hash skips the ownership assert along with the write. Only
+ // an object carrying this manifest's hash can match, and that hash covers
+ // the `clerum.io/recipe` label the assert reads; forging the annotation
+ // takes the same access as forging that label.
+ {
+ manifest,
+ readExisting: () => this.coreApi.readNamespacedConfigMap({ name, namespace: ns }),
+ }
)
}
diff --git a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
index 89e010ef0..d249886d2 100644
--- a/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
+++ b/workflow-recipes/tests/unit/workflow/workflowReconcilerBifurcation.test.ts
@@ -52,7 +52,7 @@ const mockCoreApi = {
}),
deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
- readNamespacedSecret: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
+ readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
@@ -105,6 +105,12 @@ describe('Workflow Reconciler Bifurcation', () => {
beforeEach(() => {
vi.clearAllMocks()
+ // Secrets are applied read-first, so a test that fails before its POST
+ // leaves its mock*Once queue unconsumed, and clearAllMocks keeps that queue
+ // for the next test. Reset the Secret doubles so no premise crosses tests.
+ // By default no Secret exists, the state of a freshly approved recipe.
+ mockCoreApi.createNamespacedSecret.mockReset().mockResolvedValue({})
+ mockCoreApi.readNamespacedSecret.mockReset().mockRejectedValue({ code: 404 })
mockAppsApi.readNamespacedDeployment.mockResolvedValue({
metadata: { resourceVersion: '1', generation: 1 },
spec: { replicas: 1 },
@@ -156,6 +162,8 @@ describe('Workflow Reconciler Bifurcation', () => {
})
it('continues with workload path when spec.steps is absent', async () => {
+ // The Deployment does not exist yet: the read-first gate read is a 404.
+ mockAppsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
const recipe: WorkflowRecipeCRD = {
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
@@ -172,6 +180,8 @@ describe('Workflow Reconciler Bifurcation', () => {
})
it('accepts a pre-existing recipe PVC only when its WRC ownership labels match', async () => {
+ // The Deployment does not exist yet: the read-first gate read is a 404.
+ mockAppsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
mockCoreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce({ code: 409 })
mockCoreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({
metadata: {
@@ -321,8 +331,10 @@ describe('Workflow Reconciler Bifurcation', () => {
})
it('refuses to overwrite a Secret owned by another recipe (issue #571 F1b)', async () => {
- mockCoreApi.createNamespacedSecret.mockRejectedValueOnce({ code: 409 })
- mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
+ // Every Secret name reads back as another recipe's Secret: the raw-name
+ // adoption probe declines it, and the read-first apply of the scoped name
+ // must refuse it on that read, before any write.
+ mockCoreApi.readNamespacedSecret.mockResolvedValue({
metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },
})
const recipe: WorkflowRecipeCRD = {
@@ -340,14 +352,31 @@ describe('Workflow Reconciler Bifurcation', () => {
expect(result.phase).toBe('failed')
expect(result.message).toContain('not owned by WorkflowRecipe "normal-recipe"')
+ // Liveness witness: the scoped Secret was read, so the refusal came from
+ // the read-first apply and not from an earlier failure.
+ expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
+ name: expect.stringMatching(/^normal-recipe-creds-[a-f0-9]{12}$/),
+ namespace: 'sandbox-recipes',
+ })
+ expect(mockCoreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(mockCoreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
})
it('refuses to inherit a generateKeys Secret owned by another recipe on 409 (issue #571 S3)', async () => {
- mockCoreApi.createNamespacedSecret.mockRejectedValueOnce({ code: 409 })
- mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
- metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },
+ // Race: the read-first apply finds no scoped Secret, another recipe creates
+ // it before our POST, and the POST answers 409. The re-read then returns
+ // the foreign Secret.
+ const scopedName = /^normal-recipe-gen-creds-[a-f0-9]{12}$/
+ let scopedReads = 0
+ mockCoreApi.readNamespacedSecret.mockImplementation(({ name }: { name: string }) => {
+ if (!scopedName.test(name)) return Promise.reject({ code: 404 })
+ scopedReads += 1
+ if (scopedReads === 1) return Promise.reject({ code: 404 })
+ return Promise.resolve({
+ metadata: { resourceVersion: '1', labels: { 'clerum.io/recipe': 'some-other-recipe' } },
+ })
})
+ mockCoreApi.createNamespacedSecret.mockRejectedValue({ code: 409 })
const recipe: WorkflowRecipeCRD = {
apiVersion: 'clerum.io/v1alpha1',
kind: 'WorkflowRecipe',
@@ -365,6 +394,9 @@ describe('Workflow Reconciler Bifurcation', () => {
// silently inheriting a foreign Secret.
expect(result.phase).toBe('failed')
expect(result.message).toContain('not owned by WorkflowRecipe "normal-recipe"')
+ // Liveness witness: the POST ran and the 409 was followed by a re-read.
+ expect(mockCoreApi.createNamespacedSecret).toHaveBeenCalledTimes(1)
+ expect(scopedReads).toBe(2)
})
it('rejects prepareVolumeOwnership without a non-root target UID', async () => {