Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Tests for workflowReconciler reconcile loop.
* *
* WorkflowReconciler.reconcile() signature:
* reconcile(recipeName, recipeUid, namespace, spec, currentStatus?)
*
* Deps: { coreApi, customApi, networkingApi, batchApi?, config, tokenFactory }
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'
import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'
import { captureLogger } from '../../../src/reconciler/__tests__/captureLogger'
import {
resolveStatefulSetHeadlessServiceName,
resolveWorkloadMcpServerLabel,
resolveWorkloadRuntimeResourceName,
} from '../../../src/reconciler/resourceBuilder'
import {
issueMcpHostRuntimeTokens,
issueMcpHostWorkflowControlToken,
} from '../../../src/workflow/mcpHostRuntimeTokenIssuerClient'
import { buildWorkflowNetworkPolicies } from '../../../src/workflow/networkPolicyFactory'
import type { NetworkPolicyPassSummary } from '../../../src/workflow/pluginWorkloadSdkProvisioner'
import {
buildArtifactReaderHeadlessService,
buildMcpHostHeadlessService,
buildMcpHostRouteAliasHeadlessService,
buildSnippetRunnerHeadlessService,
} from '../../../src/workflow/podFactory'
import {
buildArtifactReaderServiceName,
buildMcpHostRouteAliasServiceName,
buildMcpHostServiceName,
buildSnippetRunnerServiceName,
} from '../../../src/workflow/resourceNames'
import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'
import {
LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,
LegacyNetworkPolicyDeletePendingError,
type WorkflowRecipeSpec,
WorkflowReconciler,
type WorkflowReconcilerDeps,
buildNetworkPolicyConvergedCondition,
buildNetworkPolicyOwnershipConditions,
hasLegacyNetworkPolicyDeletePendingMarker,
networkPolicyConditionsChanged,
networkPolicyMarkerConditions,
networkPolicyMarkerFacts,
translateNetworkPolicyPassSummary,
} from '../../../src/workflow/workflowReconciler'
import { asApiserverNetworkPolicy } from './asApiserverNetworkPolicy'
vi.mock('../../../src/workflow/mcpHostRuntimeTokenIssuerClient', () => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
accessToken: 'mcp-host-runtime-access-token',
refreshToken: 'mcp-host-runtime-refresh-token',
mcpHostControlToken: 'mcp-host-workflow-control-token',
expiresInSeconds: 3600,
controlExpiresInSeconds: 3600,
}),
issueMcpHostWorkflowControlToken: vi.fn().mockResolvedValue('mcp-host-workflow-control-token'),
}))
vi.mock('../../../src/gfsBinding', () => ({
mintRecipeHostGfsToken: vi.fn().mockResolvedValue({
['to'.concat('ken')]: 'gfs-runtime-value',
expiresInSeconds: 300,
subject: 'host:3rd:sandbox-recipes/workflow-recipe',
}),
}))
// ─── Mock helpers ────────────────────────────────────────────────────────────
function makeSpec(overrides: Partial<WorkflowRecipeSpec> = {}): WorkflowRecipeSpec {
return {
agent: { provider: 'openai', model: 'gpt-4o' },
steps: [
{ id: 's1', instruction: 'Do step 1' },
{ id: 's2', instruction: 'Do step 2', dependsOn: ['s1'] },
],
...overrides,
}
}
function snippetRun(code = 'return { ok: true }') {
return { type: 'snippet' as const, language: 'typescript' as const, code }
}
function makeConfig() {
return {
sandboxNamespace: 'sandbox-recipes',
controlPlaneNamespace: 'control-plane',
mcpServerNamespace: 'mcp-server',
wrcPort: 8082,
mcpHostPort: 8080,
coordinatorImage: 'clerum/coordinator:test',
mcpHostImage: 'clerum/mcp-host:test',
artifactReaderImage: 'clerum/workflow-recipes:test',
maxWorkflowSteps: 100,
workflowDefaultRunDurationSeconds: 3600,
workflowMaxRunDurationSeconds: 86_400,
runtimeTokenTtlSeconds: 900,
runtimeTokenRefreshBeforeSeconds: 300,
runtimeEgressDnsOverlapSeconds: 300,
workflowMaxWorkloadsPerRecipe: 25,
workflowUiEgressInternalMaxItems: 25,
workflowMaxSteps: 100,
workflowStepDependsOnMaxItems: 100,
workflowStepAllowedToolsMaxItems: 50,
workflowStepMcpServersMaxItems: 20,
workflowStatefulSetMaxReplicas: 20,
workflowStatefulSetMaxVolumeClaimTemplates: 4,
workflowStatefulSetMaxPvcPreflightChecks: 80,
}
}
function makeTokenFactory() {
return {
signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('coord-mcp-token'),
signCoordinatorToWrcToken: vi.fn().mockResolvedValue('coord-wrc-token'),
// Split WRC→mcp-host signers (2026-04-09 refactor). No persistent token store.
signWrcConfigureToken: vi.fn().mockResolvedValue('wrc-configure-token'),
signWrcArtifactToken: vi.fn().mockResolvedValue('wrc-artifact-token'),
signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('wrc-artifact-delete-token'),
signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('custom-coord-wrc-token'),
}
}
function makeCoreApi(
podExists = false,
options: {
workflowOutputAnchorPhase?: string | null
workflowOutputPreparePhase?: string | null
workflowOutputPrepareWaitingReason?: string
workflowOutputPrepareSchedulingReason?: string
} = {}
) {
const createdPods = new Map<string, unknown>()
const workflowOutputPreparePhase = Object.prototype.hasOwnProperty.call(
options,
'workflowOutputPreparePhase'
)
? options.workflowOutputPreparePhase
: 'Succeeded'
const workflowOutputAnchorPhase = Object.prototype.hasOwnProperty.call(
options,
'workflowOutputAnchorPhase'
)
? options.workflowOutputAnchorPhase
: 'Running'
const readyPod = (name: string) => ({
metadata: { name },
status: {
phase: 'Running',
conditions: [{ type: 'Ready', status: 'True' }],
},
})
const workflowOutputPreparePod = (name: string) => ({
metadata: { name },
status: {
phase: workflowOutputPreparePhase ?? undefined,
conditions: [
...(options.workflowOutputPrepareSchedulingReason
? [
{
type: 'PodScheduled',
status: 'False',
reason: options.workflowOutputPrepareSchedulingReason,
},
]
: []),
],
containerStatuses: options.workflowOutputPrepareWaitingReason
? [
{
state: {
waiting: { reason: options.workflowOutputPrepareWaitingReason },
},
},
]
: [],
},
})
const workflowOutputAnchorPod = (name: string) => ({
metadata: { name },
status: {
phase: workflowOutputAnchorPhase ?? undefined,
conditions:
workflowOutputAnchorPhase === 'Running' ? [{ type: 'Ready', status: 'True' }] : [],
},
})
return {
createNamespacedSecret: vi.fn().mockResolvedValue({}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
patchNamespacedSecret: vi.fn().mockResolvedValue({}),
replaceNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
readNamespacedConfigMap: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedConfigMap: vi.fn().mockResolvedValue({}),
// Pod API on CoreV1Api
createNamespacedPod: vi.fn().mockImplementation(async ({ body }) => {
const name = body?.metadata?.name
if (name) createdPods.set(name, body)
return {}
}),
readNamespacedPod: vi.fn().mockImplementation(async ({ name }) => {
const isKnownRuntimePodName = [
'-coordinator',
'-mcp-host',
'-artifact-reader',
'-snippet-runner',
'-workflow-output-prepare',
].some(suffix => name.endsWith(suffix))
if (name.endsWith('-workflow-output-anchor') || !isKnownRuntimePodName) {
if (workflowOutputAnchorPhase === null) throw { code: 404 }
return workflowOutputAnchorPod(name)
}
if (name.endsWith('-workflow-output-prepare')) {
if (workflowOutputPreparePhase === null) throw { code: 404 }
return workflowOutputPreparePod(name)
}
if (podExists || createdPods.has(name)) return readyPod(name)
throw { code: 404 }
}),
deleteNamespacedPod: vi.fn().mockResolvedValue({}),
listNamespacedPod: vi.fn().mockResolvedValue({ items: [] }),
createNamespacedService: vi.fn().mockResolvedValue({}),
readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedService: vi.fn().mockResolvedValue({}),
deleteNamespacedService: vi.fn().mockResolvedValue({}),
deleteCollectionNamespacedService: vi.fn().mockResolvedValue({}),
// PVC API
readNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({
metadata: {
annotations: {
'volume.kubernetes.io/selected-node': 'node-a',
},
},
spec: { accessModes: ['ReadWriteOnce'] },
status: { accessModes: ['ReadWriteOnce'] },
}),
createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
}
}
function getCreatedMcpHostPod(coreApi: ReturnType<typeof makeCoreApi>) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-host'
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(`expected WRC to create test-wf-mcp-host pod; created: ${createdPodNames}`)
}
return call[0].body
}
function getCreatedCoordinatorPod(coreApi: ReturnType<typeof makeCoreApi>) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-coordinator'
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(`expected WRC to create test-wf-coordinator pod; created: ${createdPodNames}`)
}
return call[0].body
}
function getCreatedArtifactReaderPod(coreApi: ReturnType<typeof makeCoreApi>) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-artifact-reader'
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(
`expected WRC to create test-wf-artifact-reader pod; created: ${createdPodNames}`
)
}
return call[0].body
}
function getCreatedSnippetRunnerPod(coreApi: ReturnType<typeof makeCoreApi>) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(
`expected WRC to create test-wf-snippet-runner pod; created: ${createdPodNames}`
)
}
return call[0].body
}
function getCreatedWorkflowOutputAnchorPod(
coreApi: ReturnType<typeof makeCoreApi>,
name = 'test-wf-workflow-output-anchor'
) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === name
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(`expected WRC to create ${name} pod; created: ${createdPodNames}`)
}
return call[0].body
}
function getCreatedWorkflowOutputPreparePod(
coreApi: ReturnType<typeof makeCoreApi>,
name = 'test-wf-workflow-output-prepare'
) {
const call = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === name
)
if (!call) {
const createdPodNames = coreApi.createNamespacedPod.mock.calls
.map(([arg]) => arg.body?.metadata?.name)
.filter(Boolean)
.join(', ')
throw new Error(`expected WRC to create ${name} pod; created: ${createdPodNames}`)
}
return call[0].body
}
function expectWorkflowOutputAnchorAffinity(
pod: { spec?: { affinity?: any; nodeName?: string } },
claimLabel = 'test-wf-workflow-output'
): void {
const required =
pod.spec!.affinity!.podAffinity!.requiredDuringSchedulingIgnoredDuringExecution![0]
expect(pod.spec!.nodeName).toBeUndefined()
expect(required.topologyKey).toBe('kubernetes.io/hostname')
expect(required.labelSelector!.matchExpressions).toEqual([
{ key: 'clerum.io/workflow-output-claim', operator: 'In', values: [claimLabel] },
{ key: 'clerum.io/component', operator: 'In', values: ['workflow-output-anchor'] },
])
}
function makeCustomApi() {
return {
patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
getNamespacedCustomObject: vi.fn().mockResolvedValue({
metadata: { name: 'test-wf', resourceVersion: '1' },
spec: {},
status: {},
}),
createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
}
}
function makeNetworkingApi() {
return {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
}
}
/**
* Stateful NetworkPolicy double. Stored objects are what the apiserver would
* return (`asApiserverNetworkPolicy`), never the body that was sent, so a
* second pass only converges if the comparison tolerates apiserver defaults.
* create rejects 409 when present; replace rejects 404 when absent and 409 on a
* stale resourceVersion.
*/
function makeApiserverNetworkingApi() {
const live = new Map<string, k8s.V1NetworkPolicy>()
let revision = 100
const key = (namespace: string, name: string) => `${namespace}/${name}`
const store = (namespace: string, body: k8s.V1NetworkPolicy): k8s.V1NetworkPolicy => {
revision += 1
const stored = asApiserverNetworkPolicy({
...body,
metadata: { ...body.metadata, namespace },
})
stored.metadata = { ...stored.metadata, resourceVersion: String(revision) }
live.set(key(namespace, stored.metadata.name!), stored)
return structuredClone(stored)
}
const api = {
createNamespacedNetworkPolicy: vi.fn(
async ({ namespace, body }: { namespace: string; body: k8s.V1NetworkPolicy }) => {
if (live.has(key(namespace, body.metadata!.name!))) throw { code: 409 }
return store(namespace, body)
}
),
readNamespacedNetworkPolicy: vi.fn(
async ({ name, namespace }: { name: string; namespace: string }) => {
const found = live.get(key(namespace, name))
if (!found) throw { code: 404 }
return structuredClone(found)
}
),
replaceNamespacedNetworkPolicy: vi.fn(
async ({
name,
namespace,
body,
}: {
name: string
namespace: string
body: k8s.V1NetworkPolicy
}) => {
const found = live.get(key(namespace, name))
if (!found) throw { code: 404 }
if (body.metadata?.resourceVersion !== found.metadata?.resourceVersion) {
throw { code: 409 }
}
return store(namespace, body)
}
),
deleteNamespacedNetworkPolicy: vi.fn(
async ({ name, namespace }: { name: string; namespace: string }) => {
if (!live.delete(key(namespace, name))) throw { code: 404 }
return {}
}
),
listNamespacedNetworkPolicy: vi.fn(
async ({ namespace, labelSelector }: { namespace: string; labelSelector?: string }) => ({
items: [...live.values()]
.filter(
policy =>
(policy.metadata?.namespace ?? namespace) === namespace &&
policyMatchesLabelSelector(policy.metadata?.labels, labelSelector)
)
.map(policy => structuredClone(policy)),
})
),
}
return { api, live, key }
}
function policyMatchesLabelSelector(
labels: Record<string, string> | undefined,
labelSelector: string | undefined
): boolean {
if (!labelSelector) return true
const present = labels ?? {}
return labelSelector.split(',').every(term => {
const trimmed = term.trim()
const eq = trimmed.indexOf('=')
if (eq < 0) return false
return present[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)
})
}
function createdPolicyNames(networkingApi: ReturnType<typeof makeNetworkingApi>): string[] {
return networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => (arg as { body?: k8s.V1NetworkPolicy }).body?.metadata?.name ?? ''
)
}
function captureRunLaneNetworkPolicyLogs(): {
entries: Array<Record<string, unknown>>
restore: () => void
} {
const previousLevel = process.env.LOG_LEVEL
process.env.LOG_LEVEL = 'info'
const entries: Array<Record<string, unknown>> = []
const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
for (const line of text.split('\n')) {
if (!line.startsWith('{')) continue
const parsed = JSON.parse(line) as Record<string, unknown>
if (String(parsed.msg).includes('NetworkPolicy')) entries.push(parsed)
}
return true
}) as unknown as typeof process.stdout.write)
return {
entries,
restore: () => {
spy.mockRestore()
if (previousLevel === undefined) delete process.env.LOG_LEVEL
else process.env.LOG_LEVEL = previousLevel
},
}
}
function hasPublicHttpEgressRule(policy: {
spec?: {
egress?: Array<{
to?: Array<{ ipBlock?: { cidr?: string; except?: string[] } }>
ports?: Array<{ port?: number | string }>
}>
}
}): boolean {
return Boolean(
policy.spec?.egress?.some(rule => {
const ports = new Set(rule.ports?.map(port => port.port))
const requiredExcludes = [
'0.0.0.0/8',
'10.0.0.0/8',
'100.64.0.0/10',
'127.0.0.0/8',
'169.254.0.0/16',
'172.16.0.0/12',
'192.0.0.0/24',
'192.0.2.0/24',
'192.31.196.0/24',
'192.52.193.0/24',
'192.88.99.0/24',
'192.168.0.0/16',
'192.175.48.0/24',
'198.51.100.0/24',
'198.18.0.0/15',
'203.0.113.0/24',
'224.0.0.0/4',
'240.0.0.0/4',
]
return (
ports.has(443) &&
rule.to?.some(target => {
const block = target.ipBlock
return Boolean(
block?.cidr &&
(block.cidr !== '0.0.0.0/0' ||
requiredExcludes.every(item => block.except?.includes(item)))
)
})
)
})
)
}
function publicHttpEgressCidrs(policy: {
spec?: {
egress?: Array<{
to?: Array<{ ipBlock?: { cidr?: string } }>
ports?: Array<{ port?: number | string }>
}>
}
}): string[] {
return (
policy.spec?.egress
?.filter(rule => {
const ports = new Set(rule.ports?.map(port => port.port))
return ports.has(443)
})
.flatMap(rule => rule.to?.map(target => target.ipBlock?.cidr).filter(Boolean) ?? []) ?? []
)
}
function makeDeps(overrides: Partial<WorkflowReconcilerDeps> = {}): WorkflowReconcilerDeps {
return {
coreApi: makeCoreApi() as never,
customApi: makeCustomApi() as never,
networkingApi: makeNetworkingApi() as never,
config: makeConfig() as never,
tokenFactory: makeTokenFactory() as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
...overrides,
}
}
function makePgPool() {
const query = vi.fn(async (sql: string) => {
const trimmed = sql.trim().toUpperCase()
if (trimmed.startsWith('SELECT')) {
return { rows: [], rowCount: 0 }
}
return { rows: [], rowCount: 1 }
})
const release = vi.fn()
const connect = vi.fn(async () => ({ query, release }))
return {
query,
connect,
release,
pool: { query, connect } as unknown as NonNullable<WorkflowReconcilerDeps['pgPool']>,
}
}
function makeJwtWithExp(
expiresInSecondsFromNow: number,
extraClaims: Record<string, unknown> = {}
): string {
const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url')
const recipeNamespace =
typeof extraClaims.recipeNamespace === 'string'
? extraClaims.recipeNamespace
: 'recipe-runtime-ns'
const recipeName = typeof extraClaims.recipeName === 'string' ? extraClaims.recipeName : 'test-wf'
const hostRefs = Array.isArray(extraClaims.hostRefs)
? extraClaims.hostRefs
: [`${recipeNamespace}/${recipeName}`]
const payload = Buffer.from(
JSON.stringify({
exp: Math.floor(Date.now() / 1000) + expiresInSecondsFromNow,
recipeNamespace,
recipeName,
hostRefs,
...extraClaims,
})
).toString('base64url')
return `${header}.${payload}.signature`
}
function makeEncodedGfsAccess(
expiresInSecondsFromNow = 3600,
recipeName = 'test-wf',
scopes = ['gfs.read']
): string {
return Buffer.from(
makeJwtWithExp(expiresInSecondsFromNow, {
sub: `host:3rd:recipe-runtime-ns/${recipeName}`,
recipeName,
scopes,
})
).toString('base64')
}
type LiveService = ReturnType<typeof buildMcpHostHeadlessService>
function asLiveService(service: LiveService, resourceVersion: string): LiveService {
return { ...service, metadata: { ...service.metadata, resourceVersion } }
}
/** Serve `readNamespacedService` from a fixed set of live Services; 404 for any other name. */
function serveLiveServices(deps: WorkflowReconcilerDeps, live: LiveService[]): void {
const byKey = new Map(
live.map(service => [`${service.metadata?.namespace}/${service.metadata?.name}`, service])
)
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name, namespace }) => {
const service = byKey.get(`${namespace}/${name}`)
if (!service) throw { code: 404 }
return structuredClone(service)
})
}
function createdServiceNamesOf(deps: WorkflowReconcilerDeps): Array<string | undefined> {
return vi
.mocked(deps.coreApi.createNamespacedService)
.mock.calls.map(([arg]) => arg.body?.metadata?.name)
}
function serviceReadsOf(deps: WorkflowReconcilerDeps, name: string): number {
return vi
.mocked(deps.coreApi.readNamespacedService)
.mock.calls.filter(([arg]) => arg.name === name).length
}
// ─── Tests ───────────────────────────────────────────────────────────────────
describe('WorkflowReconciler — reconcile loop', () => {
let deps: WorkflowReconcilerDeps
const runId = '00000000-0000-4000-8000-000000000001'
beforeEach(() => {
vi.clearAllMocks()
deps = makeDeps()
vi.mocked(issueMcpHostRuntimeTokens).mockResolvedValue({
accessToken: 'mcp-host-runtime-access-token',
refreshToken: 'mcp-host-runtime-refresh-token',
mcpHostControlToken: 'mcp-host-workflow-control-token',
expiresInSeconds: 3600,
controlExpiresInSeconds: 3600,
})
vi.mocked(issueMcpHostWorkflowControlToken).mockResolvedValue('mcp-host-workflow-control-token')
})
afterEach(() => {
vi.unstubAllGlobals()
})
it('returns a result with a defined phase on fresh reconcile', async () => {
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(result.phase).toBeDefined()
expect(typeof result.phase).toBe('string')
})
it('registers fresh agentic parent infrastructure as active while it waits for a triggered run', async () => {
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(result.phase).toBe('active')
expect(result.workflowPhase).toBeUndefined()
expect(result.clearWorkflowExecution).toBe(true)
})
it('rejects workflow specs above configured runtime limits before creating resources', async () => {
const coreApi = makeCoreApi()
deps = makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), workflowMaxSteps: 2 } as never,
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
steps: [
{ id: 's1', instruction: 'Do step 1' },
{ id: 's2', instruction: 'Do step 2' },
{ id: 's3', instruction: 'Do step 3' },
],
})
)
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: 'spec.steps must contain at most 2 items',
})
expect(coreApi.createNamespacedConfigMap).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
})
it('maps a running workflow execution to active recipe infrastructure', async () => {
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
{
workflowExecution: { phase: 'running' },
steps: [],
},
undefined,
undefined,
runId
)
expect(result.phase).toBe('active')
expect(result.workflowPhase).toBe('running')
})
it('creates coordinator Secret via coreApi', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(deps.coreApi.createNamespacedSecret).toHaveBeenCalled()
})
it('issues mcpHost runtime tokens for the logical recipe namespace, not sandbox-recipes', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
})
it('issues mcpHost runtime tokens for the parent recipe when reconciling a child workflow run', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf-child',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
'test-wf',
runId
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
})
it('creates coordinator ConfigMap via coreApi', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(deps.coreApi.createNamespacedConfigMap).toHaveBeenCalled()
})
it('persists per-step timeout, backoff, and retry settings into the coordinator config', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
steps: [
{
id: 'render',
run: snippetRun(),
timeoutSeconds: 12,
backoffSeconds: 3,
maxRetries: 5,
},
],
})
)
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.steps[0]).toMatchObject({
id: 'render',
timeoutSeconds: 12,
backoffSeconds: 3,
maxRetries: 5,
})
})
it('creates snippet runner and artifact reader for TypeScript snippet workflows without mcp-host', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
// Shared Secret (clerum.io/shared=true) so the snippet capability passes the
// Issue #637 ownership gate — a "public-api-key" is the shared-secret case.
coreApi.readNamespacedSecret.mockResolvedValue({
metadata: { labels: { 'clerum.io/shared': 'true' } },
data: { key: 'dmFsdWU=' },
})
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
capabilities: {
secrets: [{ alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } }],
postgres: { access: 'read', workloads: ['postgres'] },
},
},
},
],
})
)
expect(result.workflowPhase).not.toBe('failed')
const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
expect(snippetRunnerPod.spec.containers[0].image).toBe('clerum/workflow-snippet-runner:test')
expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-host'
)
).toBe(false)
expect(
networkingApi.createNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg.body.metadata.name)
.filter((name: string) => name.includes('snippet-runner'))
).toEqual(
expect.arrayContaining([
'test-wf-coord-to-snippet-runner',
'test-wf-coord-to-snippet-runner-ingress',
'test-wf-snippet-runner-egress',
])
)
})
it('uses a dedicated workflow output PVC when snippet workflows declare pvc output', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
const snippetOutputMount = snippetRunnerPod.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)
const artifactReaderOutputMount = artifactReaderPod.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)
expect(snippetOutputMount).toMatchObject({
name: 'recipe-output',
mountPath: '/output',
subPath: 'workflow-output/test-wf',
})
expect(artifactReaderOutputMount).toMatchObject({
name: 'recipe-output',
mountPath: '/output',
subPath: 'workflow-output/test-wf',
})
expect(
snippetRunnerPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expect(
artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
namespace: 'sandbox-recipes',
body: expect.objectContaining({
metadata: expect.objectContaining({
name: 'test-wf-workflow-output',
labels: expect.objectContaining({
'clerum.io/recipe': 'test-wf',
'clerum.io/component': 'workflow-output',
}),
}),
spec: expect.objectContaining({
resources: { requests: { storage: '1Gi' } },
}),
}),
})
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
)
).toBe(false)
expect(result.workflowConditions).toEqual(
expect.arrayContaining([
expect.objectContaining({ type: 'WorkflowOutputPrepareGate' }),
expect.objectContaining({ type: 'WorkflowOutputWrcManagedLifecycle' }),
])
)
})
it('creates the workflow output anchor pod and waits before starting prepare/runtime pods', async () => {
const coreApi = makeCoreApi(false, {
workflowOutputAnchorPhase: null,
workflowOutputPreparePhase: null,
}) as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
expect(result.phase).toBe('deploying')
expect(result.message).toContain('Created workflow output anchor pod')
expect(getCreatedWorkflowOutputAnchorPod(coreApi)).toBeDefined()
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
)
).toBe(false)
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
)
).toBe(false)
})
it('creates a workflow output prepare pod and blocks runtime pods until it succeeds', async () => {
const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: null }) as ReturnType<
typeof makeCoreApi
>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
const preparePod = getCreatedWorkflowOutputPreparePod(coreApi)
expect(result.phase).toBe('deploying')
expect(result.message).toContain('Created workflow output prepare pod')
expect(preparePod.spec!.affinity).toBeDefined()
expect(preparePod.spec!.containers![0].env).toEqual(
expect.arrayContaining([
{ name: 'WORKFLOW_OUTPUT_SUB_PATH', value: 'workflow-output/test-wf' },
])
)
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
)
).toBe(false)
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-artifact-reader'
)
).toBe(false)
})
it('surfaces workflow output prepare pod pending state without reporting an MCP failure', async () => {
const coreApi = makeCoreApi(false, {
workflowOutputPreparePhase: 'Pending',
workflowOutputPrepareSchedulingReason: 'Unschedulable',
}) as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
})
)
expect(result.phase).toBe('deploying')
expect(result.message).toContain('Waiting for workflow output prepare pod')
expect(result.message).toContain('scheduling=Unschedulable')
expect(result.message).not.toContain('mcp-host')
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
)
).toBe(false)
})
it('recreates a failed workflow output prepare pod before starting runtime pods', async () => {
const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: 'Failed' }) as ReturnType<
typeof makeCoreApi
>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const customApi = makeCustomApi()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
customApi: customApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
}),
{
workflowExecution: { phase: 'initializing', attempt: 0 },
}
)
expect(result.phase).toBe('deploying')
expect(result.workflowPhase).toBe('recovering')
expect(result.message).toContain('recreating it (attempt 1/3)')
expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({
name: 'test-wf-workflow-output-prepare',
namespace: 'sandbox-recipes',
})
expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
expect.objectContaining({
body: {
status: {
workflowExecution: expect.objectContaining({
phase: 'recovering',
attempt: 1,
}),
},
},
}),
expect.anything()
)
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
)
).toBe(false)
})
it('fails the workflow output prepare gate after the retry budget is exhausted', async () => {
const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: 'Failed' }) as ReturnType<
typeof makeCoreApi
>
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
}),
{
workflowExecution: { phase: 'recovering', attempt: 3 },
}
)
expect(result.phase).toBe('failed')
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('prepare pod failed after 3 recovery attempts')
expect(coreApi.deleteNamespacedPod).not.toHaveBeenCalledWith({
name: 'test-wf-workflow-output-prepare',
namespace: 'sandbox-recipes',
})
})
it('uses an external workflow output PVC claim without creating or deleting it', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
name: 'external-output-claim',
labels: {
'clerum.io/workflow-output-external': 'true',
'clerum.io/workflow-output-claim': 'external-output-claim',
'clerum.io/workflow-output-scope': 'test-wf',
},
},
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', claimName: 'external-output-claim' },
})
)
const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
expect(
snippetRunnerPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('external-output-claim')
expect(
artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('external-output-claim')
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
name: 'external-output-claim',
namespace: 'sandbox-recipes',
})
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(
coreApi.createNamespacedPod.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
)
).toBe(false)
expect(result.workflowConditions).toEqual(
expect.arrayContaining([
expect.objectContaining({ type: 'WorkflowOutputExternalClaim' }),
expect.objectContaining({ type: 'WorkflowOutputRwoCompatibility' }),
])
)
})
it('rejects an external workflow output PVC claim managed by another WRC scope', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
name: 'other-wf-workflow-output',
labels: {
'clerum.io/managed-by': 'wrc',
'clerum.io/component': 'workflow-output',
'clerum.io/workflow-output-claim': 'other-wf-workflow-output',
'clerum.io/workflow-output-scope': 'other-wf',
},
},
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
output: { destination: 'pvc', claimName: 'other-wf-workflow-output' },
})
)
expect(result.phase).toBe('failed')
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('managed by WRC')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects an external workflow output PVC claim managed by the same WRC scope', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
name: 'test-wf-workflow-output',
labels: {
'clerum.io/managed-by': 'wrc',
'clerum.io/component': 'workflow-output',
'clerum.io/workflow-output-claim': 'test-wf-workflow-output',
'clerum.io/workflow-output-scope': 'test-wf',
},
},
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
output: { destination: 'pvc', claimName: 'test-wf-workflow-output' },
})
)
expect(result.phase).toBe('failed')
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('managed by WRC')
expect(result.message).toContain('remove spec.output.claimName')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects an unlabeled external workflow output PVC claim', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: { name: 'external-output-claim' },
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
output: { destination: 'pvc', claimName: 'external-output-claim' },
})
)
expect(result.phase).toBe('failed')
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('is not labeled for workflow output scope "test-wf"')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('preserves workflow output condition transition times across steady reconciles', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
name: 'external-output-claim',
labels: {
'clerum.io/workflow-output-external': 'true',
'clerum.io/workflow-output-claim': 'external-output-claim',
'clerum.io/workflow-output-scope': 'test-wf',
},
},
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
},
})
)
const previousTransition = '2026-01-01T00:00:00.000Z'
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', claimName: 'external-output-claim' },
}),
{
conditions: [
{
type: 'WorkflowOutputRwoCompatibility',
status: 'True',
reason: 'RwoOutputClaimCoLocation',
message: 'previous',
lastTransitionTime: previousTransition,
},
],
}
)
expect(
result.workflowConditions?.find(c => c.type === 'WorkflowOutputRwoCompatibility')
?.lastTransitionTime
).toBe(previousTransition)
})
it('surfaces a visible wait when an external workflow output PVC is missing', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
output: { destination: 'pvc', claimName: 'missing-output-claim' },
})
)
expect(result.phase).toBe('deploying')
expect(result.message).toContain(
'External workflow output PVC "missing-output-claim" was not found'
)
expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects a pre-existing generated workflow output PVC without matching WRC ownership labels', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.createNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 409 })
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
name: 'test-wf-workflow-output',
labels: {
'clerum.io/managed-by': 'wrc',
'clerum.io/component': 'workflow-output',
'clerum.io/workflow-output-claim': 'test-wf-workflow-output',
'clerum.io/workflow-output-scope': 'other-wf',
},
},
status: { phase: 'Bound' },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
expect(result.phase).toBe('failed')
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('not managed by WRC for workflow output scope "test-wf"')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('waits for a terminating explicit workflow output PVC before creating runtime pods', async () => {
vi.useFakeTimers()
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.createNamespacedPersistentVolumeClaim = vi
.fn()
.mockRejectedValueOnce({ code: 409 })
.mockResolvedValueOnce({})
coreApi.readNamespacedPersistentVolumeClaim = vi
.fn()
.mockResolvedValueOnce({
metadata: { deletionTimestamp: '2026-05-14T00:00:00Z' },
spec: { accessModes: ['ReadWriteOnce'] },
status: { accessModes: ['ReadWriteOnce'] },
})
.mockRejectedValueOnce({ code: 404 })
.mockResolvedValue({
metadata: { annotations: {} },
spec: { accessModes: ['ReadWriteOnce'] },
status: { accessModes: ['ReadWriteOnce'] },
})
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
const resultPromise = reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
await vi.advanceTimersByTimeAsync(1000)
const result = await resultPromise
vi.useRealTimers()
expect(result.workflowPhase).not.toBe('failed')
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(2)
expect(getCreatedSnippetRunnerPod(coreApi)).toBeDefined()
expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
})
it('recreates an explicit workflow output PVC after a Kubernetes delete race', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.createNamespacedPersistentVolumeClaim = vi
.fn()
.mockRejectedValueOnce({ code: 409 })
.mockResolvedValueOnce({})
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
},
],
output: { destination: 'pvc', storageSize: '1Gi' },
})
)
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(2)
expect(getCreatedSnippetRunnerPod(coreApi)).toBeDefined()
expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
})
it('creates artifact reader for agentic mcp-host child runs with tool-generated output artifacts', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: { provider: 'zai', model: 'glm-5-turbo' },
steps: [{ id: 'weekly-report', instruction: 'Generate a weekly summary.' }],
}),
undefined,
undefined,
undefined,
'run-123'
)
expect(result.workflowPhase).not.toBe('failed')
expect(getCreatedMcpHostPod(coreApi)).toBeDefined()
const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
expect(artifactReaderPod.spec!.containers![0].volumeMounts).toContainEqual(
expect.objectContaining({
name: 'recipe-output',
mountPath: '/output',
subPath: 'workflow-output/test-wf/run-123',
})
)
expect(
networkingApi.createNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg.body.metadata.name)
.filter((name: string) => name.includes('artifact-reader'))
).toEqual(expect.arrayContaining(['test-wf-wrc-to-artifact-reader']))
})
it('uses the parent workflow output PVC for triggered child runs with explicit pvc output', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'parent-wf-run-12345678',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: { provider: 'zai', model: 'glm-5-turbo' },
steps: [{ id: 'weekly-report', instruction: 'Generate a weekly summary.' }],
output: { destination: 'pvc', storageSize: '2Gi' },
}),
undefined,
undefined,
'parent-wf',
'run-123'
)
const mcpHostPod = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'parent-wf-run-12345678-mcp-host'
)?.[0].body
const artifactReaderPod = coreApi.createNamespacedPod.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'parent-wf-run-12345678-artifact-reader'
)?.[0].body
expect(mcpHostPod).toBeDefined()
expect(artifactReaderPod).toBeDefined()
expect(
mcpHostPod!.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('parent-wf-workflow-output')
expect(
artifactReaderPod!.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('parent-wf-workflow-output')
expect(
mcpHostPod!.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)!.subPath
).toBe('workflow-output/parent-wf/run-123')
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
namespace: 'sandbox-recipes',
body: expect.objectContaining({
metadata: expect.objectContaining({
name: 'parent-wf-workflow-output',
labels: expect.objectContaining({ 'clerum.io/recipe': 'parent-wf' }),
}),
spec: expect.objectContaining({
resources: { requests: { storage: '2Gi' } },
}),
}),
})
})
it('uses StatefulSet runtime names for snippet workload config and NetworkPolicy selectors', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const recipeName = 'manual-layer3a-api-mongo-postgres-four-step'
const recipeUid = 'statefulset-runtime-selector-uid'
const spec = makeSpec({
agent: undefined,
workloads: [
{
id: 'mongodb',
type: 'statefulset',
image: 'mongodb/mongodb-community-server:7.0-ubi8',
port: 27017,
},
{ id: 'postgres', type: 'statefulset', image: 'postgres:16-alpine', port: 5432 },
],
steps: [
{
id: 'query-mongo',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.mongo.find({ workload: "mongodb", database: "clerum", collection: "asset_prices" }, { limit: 1 })',
capabilities: { mongo: { access: 'read', workloads: ['mongodb'] } },
},
},
{
id: 'query-postgres',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
capabilities: { postgres: { access: 'read', workloads: ['postgres'] } },
},
},
],
})
const result = await reconciler.reconcile(recipeName, recipeUid, 'sandbox-recipes', spec)
expect(result.workflowPhase).not.toBe('failed')
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: { name: recipeName, namespace: 'sandbox-recipes', uid: recipeUid },
spec,
}
const mongo = spec.workloads![0]
const postgres = spec.workloads![1]
const mongoRuntimeName = resolveWorkloadRuntimeResourceName(recipeRef, mongo)
const postgresRuntimeName = resolveWorkloadRuntimeResourceName(recipeRef, postgres)
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === `${recipeName}-workflow-config`
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.workloads.find((item: { id: string }) => item.id === 'mongodb')).toMatchObject({
resourceName: mongoRuntimeName,
serviceName: resolveStatefulSetHeadlessServiceName(recipeRef, mongo),
host: `${resolveStatefulSetHeadlessServiceName(recipeRef, mongo)}.sandbox-recipes.svc.cluster.local`,
})
expect(config.workloads.find((item: { id: string }) => item.id === 'postgres')).toMatchObject({
resourceName: postgresRuntimeName,
serviceName: resolveStatefulSetHeadlessServiceName(recipeRef, postgres),
host: `${resolveStatefulSetHeadlessServiceName(recipeRef, postgres)}.sandbox-recipes.svc.cluster.local`,
})
const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === `${recipeName}-snippet-runner-egress`
)?.[0].body
const selectedApps = snippetEgress!.spec!.egress!.flatMap(rule =>
(rule.to ?? []).map(peer => peer.podSelector?.matchLabels?.app).filter(Boolean)
)
expect(selectedApps).toEqual(expect.arrayContaining([mongoRuntimeName, postgresRuntimeName]))
})
it('rejects workflows that exceed WRC_MAX_WORKFLOW_STEPS before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), maxWorkflowSteps: 2 } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
steps: [
{ id: 's1', run: snippetRun() },
{ id: 's2', run: snippetRun() },
{ id: 's3', run: snippetRun() },
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('WRC_MAX_WORKFLOW_STEPS=2')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects snippet secrets that reference platform-managed runtime Secrets', async () => {
const reconciler = new WorkflowReconciler(
makeDeps({ config: { ...makeConfig(), enableSnippetRuntime: true } as never })
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return {}',
capabilities: {
secrets: [
{
alias: 'wrc_token',
secretRef: { name: 'wf-test-wf-coordinator-token', key: 'wrc-token' },
},
],
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('cannot reference platform-managed secret')
})
it('allows snippet HTTP egress through declared allowedHosts and creates scoped public egress policy', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
expect(coreApi.createNamespacedPod).toHaveBeenCalled()
const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(snippetEgress).toBeDefined()
expect(hasPublicHttpEgressRule(snippetEgress!)).toBe(true)
expect(publicHttpEgressCidrs(snippetEgress!)).toEqual(['93.184.216.34/32'])
})
it('allows explicit public-web snippet HTTP egress without resolving exact hosts', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const resolveRuntimeHttpEgressCidrs = vi.fn()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
runtimeEgress: { http: { egressClass: 'public-web' } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://search.example.com/data")',
capabilities: {
http: { egressClass: 'public-web' },
},
},
},
],
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(snippetEgress).toBeDefined()
expect(snippetEgress!.metadata!.labels!['clerum.io/egress-class']).toBe('public-web')
expect(publicHttpEgressCidrs(snippetEgress!)).toEqual(['0.0.0.0/0'])
expect(hasPublicHttpEgressRule(snippetEgress!)).toBe(true)
})
it('fails closed when declared snippet HTTP egress cannot be resolved to allowed CIDRs', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const resolveRuntimeHttpEgressCidrs = vi
.fn()
.mockRejectedValue(new Error('runtime host resolved disallowed address'))
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('runtime host resolved disallowed address')
expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
it('refreshes runtime HTTP egress policies with DNS overlap for active snippets', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
},
},
})
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.35/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
expect(replaced!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
})
const previousExpiresAt =
replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
expect(previousExpiresAt).toBeDefined()
expect(Date.parse(previousExpiresAt!)).toBeGreaterThan(Date.now())
})
it('refreshes coordinator and snippet runtime HTTP egress policies with one overlap window', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const laterPreviousExpiresAt = new Date(Date.now() + 10 * 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
if (name === 'test-wf-coord-to-wrc') {
return {
metadata: {
resourceVersion: 'rv-coord',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.33/32',
'clerum.io/runtime-http-egress-previous-expires-at': laterPreviousExpiresAt,
},
},
}
}
if (name === 'test-wf-snippet-runner-egress') {
return {
metadata: {
resourceVersion: 'rv-snippet',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
},
},
}
}
throw { code: 404 }
})
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.36/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
enableSnippetRuntime: true,
} as never,
})
)
const spec = makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
const replacedPolicies = Object.fromEntries(
networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => [
arg.body?.metadata?.name,
arg.body,
])
)
for (const name of ['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress']) {
const policy = replacedPolicies[name]
expect(policy).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain(name)
expect(publicHttpEgressCidrs(policy!)).toEqual([
'93.184.216.33/32',
'93.184.216.34/32',
'93.184.216.35/32',
'93.184.216.36/32',
])
expect(policy!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
'clerum.io/runtime-http-egress-previous-cidrs':
'93.184.216.33/32,93.184.216.34/32,93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-expires-at': laterPreviousExpiresAt,
})
const previousExpiries = JSON.parse(
policy!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidr-expiries']
)
expect(previousExpiries['93.184.216.33/32']).toBe(laterPreviousExpiresAt)
expect(Date.parse(previousExpiries['93.184.216.34/32'])).toBeLessThan(
Date.parse(laterPreviousExpiresAt)
)
expect(Date.parse(previousExpiries['93.184.216.35/32'])).toBeLessThan(
Date.parse(laterPreviousExpiresAt)
)
}
})
it('does not extend older runtime HTTP egress CIDR expiries during a new DNS rollover', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const olderPreviousExpiresAt = new Date(Date.now() + 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
'clerum.io/runtime-http-egress-previous-expires-at': olderPreviousExpiresAt,
'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
'93.184.216.34/32': olderPreviousExpiresAt,
}),
},
},
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.36/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual([
'93.184.216.34/32',
'93.184.216.35/32',
'93.184.216.36/32',
])
const previousExpiries = JSON.parse(
replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidr-expiries']
)
expect(previousExpiries['93.184.216.34/32']).toBe(olderPreviousExpiresAt)
expect(Date.parse(previousExpiries['93.184.216.35/32'])).toBeGreaterThan(
Date.parse(olderPreviousExpiresAt)
)
})
it('prunes expired runtime HTTP egress overlap CIDRs even when DNS refresh fails', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const activePreviousExpiresAt = new Date(Date.now() + 5 * 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32,93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
'93.184.216.34/32': '2000-01-01T00:00:00.000Z',
'93.184.216.35/32': activePreviousExpiresAt,
}),
},
},
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await expect(
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
).rejects.toThrow('ENOTFOUND')
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32', '93.184.216.36/32'])
expect(replaced!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
})
})
it('leaves a terminating policy unwritten on the DNS-failure prune and still rejects with the DNS error', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const activePreviousExpiresAt = new Date(Date.now() + 5 * 60 * 1000).toISOString()
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32,93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
'93.184.216.34/32': '2000-01-01T00:00:00.000Z',
'93.184.216.35/32': activePreviousExpiresAt,
}),
},
},
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await expect(
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
).rejects.toThrow('ENOTFOUND')
expect(
networkingApi.readNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => (arg as { name: string }).name === 'test-wf-snippet-runner-egress'
).length
).toBeGreaterThanOrEqual(2)
expect(
logs.entries.some(
entry =>
entry.level === 'warn' &&
entry.reason === 'terminating' &&
String(entry.msg).includes('NetworkPolicy "test-wf-snippet-runner-egress"')
)
).toBe(true)
// The refresh rethrows the DNS error, so the prune's own error line is
// the only record that a policy still carries the CIDRs it meant to drop.
expect(
logs.entries.some(
entry =>
entry.level === 'error' &&
String(entry.msg).includes('DNS-failure prune left a NetworkPolicy pending a retry')
)
).toBe(true)
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
} finally {
logs.restore()
}
})
it('keeps the CIDRs of a policy owned by another controller out of the DNS-failure prune', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const livePolicies: Record<string, k8s.V1NetworkPolicy> = {
// Ours: the rollover to .35 is done and its .34 overlap has expired.
'test-wf-coord-to-wrc': {
metadata: {
name: 'test-wf-coord-to-wrc',
resourceVersion: 'rv-coord',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
'clerum.io/runtime-http-egress-previous-expires-at': '2000-01-01T00:00:00.000Z',
},
},
},
// Another controller's: frozen on the CIDR DNS no longer returns.
'test-wf-snippet-runner-egress': {
metadata: {
name: 'test-wf-snippet-runner-egress',
resourceVersion: 'rv-runner',
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
},
},
},
}
networkingApi.readNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name: string }) => {
const found = livePolicies[name]
if (!found) throw { code: 404 }
return structuredClone(found)
}
)
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
// A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
coordinatorImage: 'registry.example/custom-coordinator:1',
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await expect(
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
).rejects.toThrow('ENOTFOUND')
// Liveness: the prune read both policies and applied the runner, which
// the ownership branch refused.
const readNames = networkingApi.readNamespacedNetworkPolicy.mock.calls.map(
([arg]) => (arg as { name: string }).name
)
expect(readNames).toEqual(
expect.arrayContaining(['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress'])
)
expect(
logs.entries.some(
entry =>
entry.level === 'warn' &&
entry.reason === 'owner-reference-mismatch' &&
String(entry.msg).includes('test-wf-snippet-runner-egress')
)
).toBe(true)
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
expect(replaced.map(arg => arg.name)).toEqual(['test-wf-coord-to-wrc'])
expect(publicHttpEgressCidrs(replaced[0]!.body)).toEqual(['93.184.216.35/32'])
expect(replaced[0]!.body.metadata?.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
})
expect(
replaced[0]!.body.metadata?.annotations?.['clerum.io/runtime-http-egress-previous-cidrs']
).toBeUndefined()
} finally {
logs.restore()
}
})
it('creates runtime HTTP egress policies without previous overlap on first refresh', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const created = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(created).toBeDefined()
expect(publicHttpEgressCidrs(created!)).toEqual(['93.184.216.34/32'])
expect(created!.metadata!.annotations).toMatchObject({
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
})
expect(
created!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']
).toBeUndefined()
expect(
created!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
).toBeUndefined()
})
it('drops expired runtime HTTP egress overlap CIDRs on refresh', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
'clerum.io/runtime-http-egress-previous-expires-at': '2000-01-01T00:00:00.000Z',
},
},
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.35/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32'])
expect(
replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']
).toBeUndefined()
expect(
replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
).toBeUndefined()
})
it('drops invalid, private, or untrusted-future runtime HTTP egress overlap CIDRs from annotations', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: {
resourceVersion: 'rv-1',
annotations: {
'clerum.io/runtime-http-egress-current-cidrs':
'10.0.0.5/32,93.184.216.34/32,0.0.0.0/0,garbage',
'clerum.io/runtime-http-egress-previous-cidrs':
'192.168.1.5/32,93.184.216.0/24,93.184.216.33/32',
'clerum.io/runtime-http-egress-previous-expires-at': '2999-01-01T00:00:00.000Z',
},
},
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.35/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
)?.[0].body
expect(replaced).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
expect(replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']).toBe(
'93.184.216.34/32'
)
})
it('rejects snippet HTTP hosts that are not declared in runtimeEgress', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchText("https://api.example.com/data")',
capabilities: {
http: { allowedHosts: ['api.example.com'] },
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('must be declared in spec.runtimeEgress.http.allowedHosts')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects public-web snippet HTTP capabilities that also declare exact allowedHosts', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
runtimeEgress: { http: { egressClass: 'public-web' } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchText("https://api.example.com/data")',
capabilities: {
http: { egressClass: 'public-web', allowedHosts: ['api.example.com'] },
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('allowedHosts must be omitted when egressClass is public-web')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects snippet database workloads that are not declared in the recipe', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.postgres.query({ workload: "missing", database: "clerum" }, { sql: "select 1" })',
capabilities: {
postgres: { access: 'read', workloads: ['missing'] },
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('references undeclared postgres workload "missing"')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects snippet database capabilities without explicit read or readWrite access', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
capabilities: {
postgres: { workloads: ['postgres'] },
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('postgres capability must declare access read or readWrite')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects snippet MCP servers and tools outside the declared capability context', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
mcpServers: [{ id: 'mock-tools', endpoint: 'http://mock-tools.mcp-server.svc:3000/mcp' }],
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 1, b: 2 })',
capabilities: {
mcp: {
servers: ['mock-tools'],
allowedTools: { include: ['other-server__add'] },
},
},
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('must be scoped to an allowed server')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it.each([
['169.254.169.254', 'must not be an IP literal'],
['metadata.google.internal', 'must be a public DNS hostname'],
['metadata.goog', 'must be a public DNS hostname'],
['postgres.sandbox-recipes.svc.cluster.local', 'must be a public DNS hostname'],
['pod-1.sandbox-recipes.pod.cluster.local', 'must be a public DNS hostname'],
])(
'rejects non-public runtimeEgress HTTP host %s before creating workflow pods',
async (host, message) => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
runtimeEgress: { http: { allowedHosts: [host] } },
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain(message)
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
}
)
it('rejects too many runtimeEgress HTTP hosts before DNS resolution', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
resolveRuntimeHttpEgressCidrs,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
runtimeEgress: {
http: {
allowedHosts: Array.from({ length: 21 }, (_, index) => `api-${index}.example.com`),
},
},
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('runtimeEgress.http.allowedHosts must contain at most 20')
expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('replaces existing NetworkPolicies so removed HTTP egress cannot persist', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: { name: 'test-wf-coord-to-wrc', resourceVersion: 'rv-1' },
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()
const coordPolicy = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)?.[0].body
expect(coordPolicy).toBeDefined()
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
expect(coordPolicy!.metadata!.resourceVersion).toBe('rv-1')
expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(false)
})
it('prunes legacy recipe MCP server broad internet egress policies during reconcile', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
expect(result.workflowPhase).not.toBe('failed')
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'test-wf-mcp-servers-egress-internet',
namespace: 'mcp-server',
})
expect(
networkingApi.createNamespacedNetworkPolicy.mock.calls.some(
([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-servers-egress-internet'
)
).toBe(false)
networkingApi.deleteNamespacedNetworkPolicy.mockClear()
networkingApi.listNamespacedNetworkPolicy.mockClear()
const second = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
expect(second.workflowPhase).not.toBe('failed')
expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
name: 'test-wf-mcp-servers-egress-internet',
namespace: 'mcp-server',
})
})
// R4-L4: a 403 does not mark the policy gone; the next DELETE waits for the
// end of its backoff window (60 s after the first failure).
it('G3: does not skip the legacy internet NP after DELETE 403, and retries it once its backoff window ends', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {
throw { code: 403 }
}
return {}
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
const legacyDeletes = () =>
networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) =>
arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
)
const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
vi.useFakeTimers({ toFake: ['Date'] })
try {
vi.setSystemTime(t0)
const first = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
expect(legacyDeletes()).toHaveLength(1)
expect(
(first.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
)
).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
vi.setSystemTime(t0 + 30_000)
const inside = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
// Liveness witness: the pass reached the prune (the legacy fact is
// published), and inside the window it sent no DELETE.
expect(
(inside.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
)
).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
expect(legacyDeletes()).toHaveLength(1)
expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
30_000
)
vi.setSystemTime(t0 + 60_000)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
expect(legacyDeletes()).toHaveLength(2)
// The second failure doubles the window.
expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
120_000
)
} finally {
vi.useRealTimers()
}
})
it('R4-L4: doubles the legacy DELETE backoff up to 1 h, and a 404 ends it', async () => {
const LEGACY = 'test-wf-mcp-servers-egress-internet'
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
let legacyAnswer = { code: 403 }
networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
if (name === LEGACY && namespace === 'mcp-server') throw legacyAnswer
return {}
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
const legacyDeletes = () =>
networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === LEGACY && arg.namespace === 'mcp-server'
)
vi.useFakeTimers({ toFake: ['Date'] })
try {
let now = new Date('2026-09-23T10:00:00.000Z').getTime()
const windows: number[] = []
for (let attempt = 0; attempt < 8; attempt++) {
vi.setSystemTime(now)
expect(
await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')
).toBe('pending')
const delay = reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')
windows.push(delay)
now += delay
}
expect(windows).toEqual([
60_000, 120_000, 240_000, 480_000, 960_000, 1_920_000, 3_600_000, 3_600_000,
])
expect(legacyDeletes()).toHaveLength(8)
legacyAnswer = { code: 404 }
vi.setSystemTime(now)
expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
'removed'
)
expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(0)
expect(legacyDeletes()).toHaveLength(9)
// Observed gone: a later retry answers from the gone set, with no DELETE.
expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
'removed'
)
expect(legacyDeletes()).toHaveLength(9)
} finally {
vi.useRealTimers()
}
})
it('R4-L4: the finalizer deletes the unlabelled legacy internet policy by name in the mcp-server namespace', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
// The label sweep finds nothing: the legacy policy predates the labels.
networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes', makeSpec(), 'uid-123')
// Liveness witness: the fixed run-lane list was deleted by name.
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'test-wf-mcp-host-to-servers',
namespace: 'sandbox-recipes',
})
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'test-wf-mcp-servers-egress-internet',
namespace: 'mcp-server',
})
})
it('clears the legacy internet NP process set on recipe delete so a recreate can delete again', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
networkingApi.deleteNamespacedNetworkPolicy.mockClear()
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes', makeSpec(), 'uid-123')
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'test-wf-mcp-servers-egress-internet',
namespace: 'mcp-server',
})
})
it('keys the legacy internet NP ledger by recipe uid, so a recreated recipe deletes again', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
})
)
const legacyDeletes = () =>
networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) =>
arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
)
const reconcileWithUid = (uid: string) =>
reconciler.reconcile(
'test-wf',
uid,
'sandbox-recipes',
makeSpec({ mcpServers: ['redis-mcp'] })
)
await reconcileWithUid('uid-a')
await reconcileWithUid('uid-a')
// Witness: the ledger suppresses the repeat for the same uid.
expect(legacyDeletes()).toHaveLength(1)
// Same name, new uid, and no reconcileDelete seen by this process (another
// replica handled the delete): a new identity must not be suppressed.
await reconcileWithUid('uid-b')
expect(legacyDeletes()).toHaveLength(2)
})
describe('B2 desired-set NetworkPolicy prune', () => {
const RECIPE = 'test-wf'
const SANDBOX = 'sandbox-recipes'
const TWO_TERM_SELECTOR = `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`
const CODEX_PROXY = `${RECIPE}-mcp-host-to-codex-proxy`
const GROK_PROXY = `${RECIPE}-mcp-host-to-grok-proxy`
const COORD_TO_WRC = `${RECIPE}-coord-to-wrc`
const COORD_TO_MCP_HOST = `${RECIPE}-coord-to-mcp-host`
const WRC_LABELS = {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'wrc',
}
function ineligibleProjection(
overrides: Partial<CodexExecutionProjection> = {}
): CodexExecutionProjection {
return {
targets: [],
eligibleTargets: [],
derivedScopes: [],
requiresCodexProxyEgress: false,
driftHashInput: '',
catalogContentHash: null,
catalogRevision: null,
connectionRevision: null,
eligibility: 'ineligible',
reason: 'static_only',
...overrides,
}
}
function seedLivePolicy(
apiserver: ReturnType<typeof makeApiserverNetworkingApi>,
name: string,
labels: Record<string, string>,
namespace = SANDBOX
): void {
apiserver.live.set(apiserver.key(namespace, name), {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: { name, namespace, labels },
spec: { podSelector: {}, policyTypes: ['Egress'] },
})
}
function listCalls(
api: ReturnType<typeof makeApiserverNetworkingApi>['api']
): Array<{ namespace: string; labelSelector?: string }> {
return api.listNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
}
function sandboxDeletes(
api: ReturnType<typeof makeApiserverNetworkingApi>['api']
): Array<{ name: string; namespace: string }> {
return api.deleteNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg)
.filter(arg => arg.namespace === SANDBOX)
}
async function applyPolicies(
reconciler: WorkflowReconciler,
opts: {
codex?: Partial<CodexExecutionProjection>
grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }
awaitsTriggeredRun?: boolean
eagerSdkMcpHost?: boolean
} = {}
): Promise<{ retryPending: boolean }> {
const spec = makeSpec()
const runtime = deriveWorkflowRuntimePlan(spec, {
recipeName: RECIPE,
runtimeScopeRecipeName: RECIPE,
workflowRunId: opts.awaitsTriggeredRun ? undefined : 'run-1',
})
const codex = ineligibleProjection({
requiresCodexProxyEgress: false,
...opts.codex,
})
const grok = {
...ineligibleProjection(),
requiresCodexProxyEgress: false,
requiresGrokProxyEgress: false,
...opts.grok,
}
return (await (
reconciler as unknown as {
applyWorkflowNetworkPolicies: (
recipeName: string,
recipeUid: string,
spec: WorkflowRecipeSpec,
runtime: ReturnType<typeof deriveWorkflowRuntimePlan>,
awaitsTriggeredRun: boolean,
codexProjection: CodexExecutionProjection,
eagerSdkMcpHost: boolean,
grokProjection: typeof grok
) => Promise<{ retryPending: boolean }>
}
).applyWorkflowNetworkPolicies(
RECIPE,
'uid-123',
spec,
runtime,
opts.awaitsTriggeredRun === true,
codex,
opts.eagerSdkMcpHost === true,
grok
)) as { retryPending: boolean }
}
function expectListWitness(
api: ReturnType<typeof makeApiserverNetworkingApi>['api'],
times = 1
): void {
expect(listCalls(api)).toEqual(
Array.from({ length: times }, () => ({
namespace: SANDBOX,
labelSelector: TWO_TERM_SELECTOR,
}))
)
}
/**
* Liveness witness for the negative prune tests: a catalog member that is not
* desired sits in the same LIST and must be deleted. If the prune loop never
* ran, "X was not deleted" would hold anyway; the sibling's delete proves the
* loop iterated over the listed items.
*/
function seedPrunableSibling(apiserver: ReturnType<typeof makeApiserverNetworkingApi>): void {
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
}
function expectPrunableSiblingDeleted(
apiserver: ReturnType<typeof makeApiserverNetworkingApi>
): void {
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
}
it('does not delete a listed name that is outside the run-lane catalog', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
})
it('G1: reserved coordinator-to-gfs survives a run-lane apply', async () => {
const apiserver = makeApiserverNetworkingApi()
const gfsName = `${RECIPE}-coordinator-to-gfs`
seedLivePolicy(apiserver, gfsName, WRC_LABELS)
seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'workflow-recipes',
})
seedLivePolicy(apiserver, `${RECIPE}-outer-unknown-wrc`, WRC_LABELS)
seedLivePolicy(apiserver, 'other-recipe-coordinator-to-gfs', {
'clerum.io/recipe': 'other-recipe',
'clerum.io/managed-by': 'wrc',
})
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
expect(apiserver.live.has(apiserver.key(SANDBOX, gfsName))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-outer-unknown-wrc`))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, 'other-recipe-coordinator-to-gfs'))).toBe(
true
)
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
})
it('deletes a seeded leftover Grok proxy and does not name-delete when unseeded', async () => {
const seeded = makeApiserverNetworkingApi()
seedLivePolicy(seeded, GROK_PROXY, WRC_LABELS)
const seededReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: seeded.api as never })
)
await applyPolicies(seededReconciler)
expectListWitness(seeded.api)
expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([GROK_PROXY])
const unseeded = makeApiserverNetworkingApi()
const unseededReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: unseeded.api as never })
)
await applyPolicies(unseededReconciler)
expectListWitness(unseeded.api)
expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(GROK_PROXY)
})
it('deletes a seeded leftover Codex proxy and does not name-delete when unseeded', async () => {
const seeded = makeApiserverNetworkingApi()
seedLivePolicy(seeded, CODEX_PROXY, WRC_LABELS)
const seededReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: seeded.api as never })
)
await applyPolicies(seededReconciler)
expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([CODEX_PROXY])
const unseeded = makeApiserverNetworkingApi()
const unseededReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: unseeded.api as never })
)
await applyPolicies(unseededReconciler)
expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(CODEX_PROXY)
})
it('c2: awaiting-trigger prunes a run-only leftover that apply(false) would keep', async () => {
const leftover = `${RECIPE}-wrc-to-artifact-reader`
const awaiting = makeApiserverNetworkingApi()
seedLivePolicy(awaiting, leftover, WRC_LABELS)
const awaitingReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: awaiting.api as never })
)
await applyPolicies(awaitingReconciler, { awaitsTriggeredRun: true })
expect(sandboxDeletes(awaiting.api).map(d => d.name)).toEqual([leftover])
const running = makeApiserverNetworkingApi()
seedLivePolicy(running, leftover, WRC_LABELS)
const runningReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: running.api as never })
)
await applyPolicies(runningReconciler, { awaitsTriggeredRun: false })
expect(sandboxDeletes(running.api).map(d => d.name)).not.toContain(leftover)
expect(running.live.has(running.key(SANDBOX, leftover))).toBe(true)
})
it('does not delete in the sandbox namespace when the LIST is empty', async () => {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api)).toEqual([])
})
it('does not delete a listed policy that is in the desired set', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
})
it('does not prune an uncatalogued policy carrying the recipe label pair; LIST is issued once', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'workflow-recipes',
})
seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler)
expectListWitness(apiserver.api)
expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(
`${RECIPE}-oauth-broker-egress`
)
expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
})
it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {
const uncertain = makeApiserverNetworkingApi()
seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)
const debugLog = captureLogger('debug')
try {
const uncertainReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: uncertain.api as never })
)
await applyPolicies(uncertainReconciler, {
codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },
})
// The skip names why the proxy survives.
expect(debugLog).toHaveBeenCalledWith(
'Skipping run-lane NP prune',
expect.objectContaining({ policy: CODEX_PROXY, reason: 'uncertain' })
)
} finally {
debugLog.mockRestore()
}
expectListWitness(uncertain.api)
expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)
expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)
const decided = makeApiserverNetworkingApi()
seedLivePolicy(decided, CODEX_PROXY, WRC_LABELS)
const decidedReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: decided.api as never })
)
await applyPolicies(decidedReconciler, {
codex: { eligibility: 'ineligible', requiresCodexProxyEgress: false },
})
expectListWitness(decided.api)
expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(CODEX_PROXY)
})
it('keeps a desired live Codex proxy and deletes an undesired Grok proxy only on the first pass', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, CODEX_PROXY, WRC_LABELS)
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler, {
codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
})
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
expect(apiserver.live.has(apiserver.key(SANDBOX, CODEX_PROXY))).toBe(true)
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
apiserver.api.listNamespacedNetworkPolicy.mockClear()
apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
await applyPolicies(reconciler, {
codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
})
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api)).toEqual([])
})
it('keeps an unlisted-as-desired Grok proxy while eligibility is uncertain, and deletes the twin', async () => {
const uncertain = makeApiserverNetworkingApi()
seedLivePolicy(uncertain, GROK_PROXY, WRC_LABELS)
const uncertainReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: uncertain.api as never })
)
await applyPolicies(uncertainReconciler, {
grok: { eligibility: 'uncertain', requiresGrokProxyEgress: false, reason: 'forbidden' },
})
expectListWitness(uncertain.api)
expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(GROK_PROXY)
expect(uncertain.live.has(uncertain.key(SANDBOX, GROK_PROXY))).toBe(true)
const decided = makeApiserverNetworkingApi()
seedLivePolicy(decided, GROK_PROXY, WRC_LABELS)
const decidedReconciler = new WorkflowReconciler(
makeDeps({ networkingApi: decided.api as never })
)
await applyPolicies(decidedReconciler, {
grok: { eligibility: 'ineligible', requiresGrokProxyEgress: false },
})
expectListWitness(decided.api)
expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(GROK_PROXY)
})
it('does not prune a catalog-named recipes-lane policy (two-term selector)', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, COORD_TO_MCP_HOST, {
'clerum.io/recipe': RECIPE,
'clerum.io/managed-by': 'workflow-recipes',
})
seedPrunableSibling(apiserver)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler, { awaitsTriggeredRun: true })
expectListWitness(apiserver.api)
expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
})
it('deletes a leftover coord-to-mcp-host when awaiting trigger', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler, { awaitsTriggeredRun: true })
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(COORD_TO_MCP_HOST)
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(false)
})
it('R1-M1: does not prune a catalog policy that carries an ownerReference (apply refuses it too)', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)
const seeded = apiserver.live.get(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))!
seeded.metadata = {
...seeded.metadata,
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'someone-else',
uid: 'uid-deployment',
controller: true,
},
],
}
seedPrunableSibling(apiserver)
const warnLog = captureLogger('warn')
try {
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
await applyPolicies(reconciler, { awaitsTriggeredRun: true })
expectListWitness(apiserver.api)
// Liveness witness: the WRC-owned sibling in the same LIST was pruned.
expectPrunableSiblingDeleted(apiserver)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
expect(warnLog).toHaveBeenCalledWith('Skipping run-lane NP prune', {
recipe: RECIPE,
policy: COORD_TO_MCP_HOST,
reason: 'owner-reference-mismatch',
})
} finally {
warnLog.mockRestore()
}
})
it('eager SDK path still prunes a leftover Grok proxy', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
await applyPolicies(reconciler, { awaitsTriggeredRun: true, eagerSdkMcpHost: true })
expectListWitness(apiserver.api)
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
})
// A prune DELETE that did not land is prune-pending, not a pending retry:
// it publishes PrunePending and does not requeue (R3-L3).
it('marks the prune pending, not a retry, when a leftover prune DELETE fails', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name, namespace }: { name: string; namespace: string }) => {
if (name === GROK_PROXY) throw { code: 403, message: 'forbidden' }
if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
expect(summary.prune).toBe('pending')
expect(summary.retryPending).toBe(false)
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
})
it.each([
{ code: 404, prune: 'converged' },
{ code: 409, prune: 'pending' },
{ code: 500, prune: 'pending' },
])('prune DELETE answering $code leaves the prune $prune', async ({ code, prune }) => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name, namespace }: { name: string; namespace: string }) => {
if (name === GROK_PROXY) throw { code }
if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
// Liveness witness: the prune reached the DELETE for the leftover.
expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
expect(summary.prune).toBe(prune)
expect(summary.retryPending).toBe(false)
})
it('logs prune failures as an error object, never as a pre-flattened message string', async () => {
const errorLog = captureLogger('error')
try {
// A client exception whose message embeds the response body and headers: the
// logger only redacts those when it receives the error itself.
const failure = Object.assign(new Error('HTTP-Code: 403 Body: "{}" Headers: {}'), {
code: 403,
})
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name, namespace }: { name: string; namespace: string }) => {
if (name === GROK_PROXY) throw failure
if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
return {}
}
)
apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce(failure)
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
await applyPolicies(reconciler) // LIST fails
await applyPolicies(reconciler) // LIST ok, DELETE of the leftover fails
const fields = (message: string) =>
errorLog.mock.calls.find(([msg]) => msg === message)?.[1] as
| Record<string, unknown>
| undefined
for (const message of [
'Run-lane NP prune LIST failed; apply outcome is kept',
'Run-lane NP prune delete failed',
]) {
expect(fields(message), message).toBeDefined()
expect(fields(message)?.err).toBe(failure)
expect(fields(message)).not.toHaveProperty('error')
}
} finally {
errorLog.mockRestore()
}
})
it('marks the legacy policy pending, apart from the prune, when the legacy internet NP DELETE fails', async () => {
const apiserver = makeApiserverNetworkingApi()
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name, namespace }: { name: string; namespace: string }) => {
if (name === `${RECIPE}-mcp-servers-egress-internet`) {
throw { code: 403, message: 'forbidden' }
}
if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
// Liveness witness: the legacy delete was attempted.
expect(
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
)
).toHaveLength(1)
// R4-L3: the legacy delete is its own fact; the catalog prune converged.
expect(summary.legacy).toBe('pending')
expect(summary.prune).toBe('converged')
expect(summary.retryPending).toBe(false)
})
// R4-L6: a failed LIST evaluated nothing. It is not a DELETE that did
// not land, so it is its own prune fact, `unevaluated`.
it('R4-L6: marks the prune unevaluated, not pending, and keeps the apply outcome when the prune LIST fails', async () => {
const apiserver = makeApiserverNetworkingApi()
seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
apiserver.api.listNamespacedNetworkPolicy.mockRejectedValue({ code: 500, message: 'boom' })
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))
const summary = await applyPolicies(reconciler)
expect(summary.prune).toBe('unevaluated')
expect(summary.retryPending).toBe(false)
expect(sandboxDeletes(apiserver.api)).toEqual([])
expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
// The apply outcome survives: the legacy internet policy prune that runs
// after the catalog prune is still attempted despite the failed LIST.
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalled()
expect(
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
)
).toHaveLength(1)
})
})
describe('B3(d) skip coordinator DELETE after same-pass 404', () => {
const COORDINATOR = 'test-wf-coordinator'
const MCP_HOST = 'test-wf-mcp-host'
const SANDBOX = 'sandbox-recipes'
function podReads(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
return coreApi.readNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
}
function podDeletes(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
}
it('does not delete the coordinator when same-pass GET is 404', async () => {
const coreApi = makeCoreApi(false)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
expect(result.workflowPhase).not.toBe('failed')
expect(podReads(coreApi, COORDINATOR)).toBe(2)
expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
})
it('G4: deletes the coordinator when GET 200 has an empty status.phase', async () => {
const coreApi = makeCoreApi(false)
coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
if (name === COORDINATOR) return { metadata: { name }, status: {} }
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
expect(podReads(coreApi, COORDINATOR)).toBe(2)
expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
})
it('G4: a 403 on coordinator GET does not skip as 404', async () => {
const coreApi = makeCoreApi(false)
coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
if (name === COORDINATOR) throw { code: 403, message: 'forbidden' }
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
expect(result.workflowPhase).toBe('failed')
// The 403 stops the reconcile at the first coordinator read; a 403
// mapped to absent would continue to a second read.
expect(podReads(coreApi, COORDINATOR)).toBe(1)
expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
})
it('c2: does not delete a Running coordinator when workflowRunId is set', async () => {
const coreApi = makeCoreApi(true)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
SANDBOX,
makeSpec(),
undefined,
undefined,
undefined,
'run-1'
)
expect(podReads(coreApi, COORDINATOR)).toBe(3)
expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
})
it('deletes the coordinator when same-pass getPodPhase is Running', async () => {
const coreApi = makeCoreApi(true)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
expect(podReads(coreApi, COORDINATOR)).toBe(3)
expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({
name: COORDINATOR,
namespace: SANDBOX,
})
})
it('neither reads nor deletes mcp-host when needsMcpHost is false', async () => {
const coreApi = makeCoreApi(false)
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
SANDBOX,
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
})
)
expect(podReads(coreApi, MCP_HOST)).toBe(0)
expect(podDeletes(coreApi, MCP_HOST)).toBe(0)
// Snippet-only has needsMcpHost=false, so awaitsTriggeredRun is false and
// B3(d) does not run. The coordinator is still read for crash recovery.
expect(podReads(coreApi, COORDINATOR)).toBe(2)
expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
})
it('does not delete mcp-host when same-pass GET is 404', async () => {
const coreApi = makeCoreApi(false)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())
expect(result.workflowPhase).not.toBe('failed')
expect(podReads(coreApi, MCP_HOST)).toBe(2)
expect(podDeletes(coreApi, MCP_HOST)).toBe(0)
})
})
it('retries NetworkPolicy replace once when resourceVersion is stale', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const readCounts = new Map<string, number>()
const coordResourceVersions: Array<string | undefined> = []
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
const count = readCounts.get(name) ?? 0
readCounts.set(name, count + 1)
return {
metadata: {
name,
resourceVersion: name === 'test-wf-coord-to-wrc' && count === 1 ? 'rv-2' : 'rv-1',
},
}
})
networkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {
if (name === 'test-wf-coord-to-wrc') {
coordResourceVersions.push(body.metadata?.resourceVersion)
const coordCalls = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
).length
if (coordCalls === 1) throw { code: 409 }
}
return {}
})
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: networkingApi as never,
config: { ...makeConfig() } as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'prepare', run: snippetRun() }],
})
)
expect(result.workflowPhase).not.toBe('failed')
const coordCalls = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
expect(coordCalls).toHaveLength(2)
expect(coordResourceVersions).toEqual(['rv-1', 'rv-2'])
expect(readCounts.get('test-wf-coord-to-wrc')).toBe(2)
expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
})
describe('run-lane NetworkPolicy convergence against apiserver-shaped live objects', () => {
const RUN_LANE_POLICY_NAMES = [
'sandbox-recipes/test-wf-coord-to-snippet-runner',
'sandbox-recipes/test-wf-coord-to-snippet-runner-ingress',
'sandbox-recipes/test-wf-coord-to-wrc',
'sandbox-recipes/test-wf-snippet-runner-egress',
'sandbox-recipes/test-wf-wrc-to-artifact-reader',
]
function readPolicyNames(api: ReturnType<typeof makeApiserverNetworkingApi>['api']): string[] {
return api.readNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name).sort()
}
function runtimeEgressSpec(): WorkflowRecipeSpec {
return makeSpec({
agent: undefined,
runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
capabilities: { http: { allowedHosts: ['api.example.com'] } },
},
},
],
})
}
it('writes nothing on a second pass once every policy is live', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
expect([...live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)
api.createNamespacedNetworkPolicy.mockClear()
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(second.workflowPhase).not.toBe('failed')
const expectedNames = RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
expect(readPolicyNames(api)).toEqual(expectedNames)
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
for (const name of expectedNames) {
expect(
logs.entries.some(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
)
).toBe(true)
}
} finally {
logs.restore()
}
})
// The factory always sends policyTypes and port protocols, so the reconcile
// tests above never depend on the apiserver default tolerance. This drives the
// run-lane apply directly with a policy that omits both.
it('leaves a policy unchanged when the apiserver filled the defaults the desired omits', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const applyNetworkPolicy = (
reconciler as unknown as {
applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<unknown>
}
).applyNetworkPolicy.bind(reconciler)
const name = 'test-wf-defaults-omitted'
const desired = (): k8s.V1NetworkPolicy => ({
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name,
namespace: 'sandbox-recipes',
labels: { 'clerum.io/managed-by': 'wrc', 'clerum.io/recipe': 'test-wf' },
},
spec: {
podSelector: { matchLabels: { app: 'test-wf-coordinator' } },
egress: [{ ports: [{ port: 443 }] }],
},
})
expect(await applyNetworkPolicy(desired())).toEqual({ policy: name, action: 'created' })
const stored = live.get(key('sandbox-recipes', name))
expect(stored?.spec?.policyTypes).toEqual(['Ingress', 'Egress'])
expect(stored?.spec?.egress?.[0]?.ports?.[0]?.protocol).toBe('TCP')
api.createNamespacedNetworkPolicy.mockClear()
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
expect(await applyNetworkPolicy(desired())).toEqual({ policy: name, action: 'unchanged' })
expect(readPolicyNames(api)).toEqual([name])
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(
logs.entries.filter(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
)
).toHaveLength(1)
} finally {
logs.restore()
}
})
it('replaces exactly the drifted policy with its live resourceVersion', async () => {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
const driftedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
const drifted = live.get(driftedKey)
expect(drifted).toBeDefined()
drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { app: 'edited-by-hand' } } }
const liveResourceVersion = drifted!.metadata!.resourceVersion
api.createNamespacedNetworkPolicy.mockClear()
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(second.workflowPhase).not.toBe('failed')
expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const [replaceArg] = api.replaceNamespacedNetworkPolicy.mock.calls[0]!
expect(replaceArg.name).toBe('test-wf-coord-to-wrc')
expect(replaceArg.body.metadata?.resourceVersion).toBe(liveResourceVersion)
expect(live.get(driftedKey)!.spec!.podSelector).not.toEqual({
matchLabels: { app: 'edited-by-hand' },
})
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
it('refresh leaves a policy owned by another controller untouched and warns', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: api as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = runtimeEgressSpec()
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const foreign = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
expect(foreign).toBeDefined()
foreign!.metadata = {
...foreign!.metadata,
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
}
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
await expect(
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
).resolves.toEqual({
conflicts: [
{ policy: 'test-wf-snippet-runner-egress', reason: 'owner-reference-mismatch' },
],
retryPending: false,
})
expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-snippet-runner-egress'
)
).toHaveLength(0)
// Only the ownership branch emits this reason; a vanished-after-conflict
// warning also names the policy and must not satisfy this witness.
expect(
logs.entries.some(
entry =>
entry.level === 'warn' &&
entry.reason === 'owner-reference-mismatch' &&
String(entry.msg).includes('test-wf-snippet-runner-egress')
)
).toBe(true)
} finally {
logs.restore()
}
})
it('refresh returns a pending retry for a terminating policy and leaves it unwritten', async () => {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: api as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = runtimeEgressSpec()
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const terminating = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
expect(terminating).toBeDefined()
terminating!.metadata = {
...terminating!.metadata,
deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
}
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
await expect(
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
).resolves.toEqual({ conflicts: [], retryPending: true })
expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-snippet-runner-egress'
)
).toHaveLength(0)
})
describe('runtime HTTP egress resolved-at', () => {
// runtimeEgressSpec has no coordinator, so the snippet runner is the
// only runtime HTTP egress policy.
const RUNTIME_EGRESS_POLICIES = ['test-wf-snippet-runner-egress']
const RESOLVED_AT = 'clerum.io/runtime-http-egress-resolved-at'
const FIRST_PASS_AT = new Date('2026-09-23T10:00:00.000Z')
const SECOND_PASS_AT = new Date('2026-09-23T10:05:00.000Z')
afterEach(() => {
vi.useRealTimers()
})
async function refreshTwice(resolveRuntimeHttpEgressCidrs: ReturnType<typeof vi.fn>) {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: apiserver.api as never,
resolveRuntimeHttpEgressCidrs,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = runtimeEgressSpec()
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(FIRST_PASS_AT)
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
for (const name of RUNTIME_EGRESS_POLICIES) {
expect(
apiserver.live.get(apiserver.key('sandbox-recipes', name))?.metadata?.annotations?.[
RESOLVED_AT
]
).toBe(FIRST_PASS_AT.toISOString())
}
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
vi.setSystemTime(SECOND_PASS_AT)
const second = await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
return { ...apiserver, second }
}
it('keeps the live resolved-at and writes nothing when the resolved set is unchanged', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const resolve = vi.fn().mockResolvedValue(['93.184.216.34/32'])
const { api, live, key, second } = await refreshTwice(resolve)
expect(resolve).toHaveBeenCalledTimes(2)
expect(readPolicyNames(api)).toEqual(expect.arrayContaining(RUNTIME_EGRESS_POLICIES))
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
for (const name of RUNTIME_EGRESS_POLICIES) {
expect(
live.get(key('sandbox-recipes', name))?.metadata?.annotations?.[RESOLVED_AT]
).toBe(FIRST_PASS_AT.toISOString())
}
// No write is also what a deferred retry produces. The Unchanged line,
// which the first pass cannot emit because it creates the policy, and
// the empty summary prove the second pass decided the policy converged.
expect(second).toEqual({ conflicts: [], retryPending: false })
for (const name of RUNTIME_EGRESS_POLICIES) {
expect(
logs.entries.filter(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
)
).toHaveLength(1)
}
} finally {
logs.restore()
}
})
it('stamps the current time on a policy that is missing although its sibling is unchanged', async () => {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: apiserver.api as never,
resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
// A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
const spec: WorkflowRecipeSpec = {
...runtimeEgressSpec(),
coordinatorImage: 'registry.example/custom-coordinator:1',
}
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(FIRST_PASS_AT)
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const coordKey = apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
const runnerKey = apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress')
for (const policyKey of [coordKey, runnerKey]) {
expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(
FIRST_PASS_AT.toISOString()
)
}
expect(apiserver.live.delete(coordKey)).toBe(true)
apiserver.api.createNamespacedNetworkPolicy.mockClear()
vi.setSystemTime(SECOND_PASS_AT)
await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
// The unchanged sibling must not lend its old resolved-at to a policy
// that did not exist: the recreated one carries the current time.
const created = apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
expect(created.map(arg => arg.body.metadata?.name)).toEqual(['test-wf-coord-to-wrc'])
expect(created[0]!.body.metadata?.annotations?.[RESOLVED_AT]).toBe(
SECOND_PASS_AT.toISOString()
)
expect(apiserver.live.get(coordKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(
SECOND_PASS_AT.toISOString()
)
})
it('stamps the current time when the resolved set changes', async () => {
const resolve = vi
.fn()
.mockResolvedValueOnce(['93.184.216.34/32'])
.mockResolvedValueOnce(['93.184.216.35/32'])
const { api } = await refreshTwice(resolve)
const replaced = api.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
expect(replaced.map(arg => arg.name).sort()).toEqual(RUNTIME_EGRESS_POLICIES)
for (const arg of replaced) {
expect(arg.body.metadata?.annotations).toMatchObject({
[RESOLVED_AT]: SECOND_PASS_AT.toISOString(),
'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
})
}
})
it('keeps the live resolved-at through a DNS-failure prune, so later failures and a healthy refresh write nothing', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const apiserver = makeApiserverNetworkingApi()
const resolve = vi
.fn()
.mockResolvedValueOnce(['93.184.216.34/32'])
.mockResolvedValueOnce(['93.184.216.35/32'])
.mockRejectedValueOnce(new Error('ENOTFOUND'))
.mockRejectedValueOnce(new Error('ENOTFOUND'))
.mockResolvedValueOnce(['93.184.216.35/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: apiserver.api as never,
resolveRuntimeHttpEgressCidrs: resolve,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const spec = runtimeEgressSpec()
const refresh = () =>
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const clearCalls = () => {
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
}
const liveAnnotations = (name: string) =>
apiserver.live.get(apiserver.key('sandbox-recipes', name))?.metadata?.annotations
// Two healthy passes leave the live policy with a resolved-at and a
// previous CIDR whose overlap window is still open.
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(FIRST_PASS_AT)
await refresh()
vi.setSystemTime(SECOND_PASS_AT)
await refresh()
for (const name of RUNTIME_EGRESS_POLICIES) {
expect(liveAnnotations(name)).toMatchObject({
[RESOLVED_AT]: SECOND_PASS_AT.toISOString(),
'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
})
}
// DNS fails after the overlap expired: the prune drops the expired
// CIDR in one PUT per policy and carries the live resolved-at over.
const pruneAt = new Date(SECOND_PASS_AT.getTime() + 24 * 60 * 60 * 1000)
clearCalls()
vi.setSystemTime(pruneAt)
await expect(refresh()).rejects.toThrow('ENOTFOUND')
const pruned = apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
expect(pruned.map(arg => arg.name).sort()).toEqual(RUNTIME_EGRESS_POLICIES)
for (const arg of pruned) {
expect(arg.body.metadata?.annotations?.[RESOLVED_AT]).toBe(SECOND_PASS_AT.toISOString())
expect(
arg.body.metadata?.annotations?.['clerum.io/runtime-http-egress-previous-cidrs']
).toBeUndefined()
}
// A second DNS failure reads the policies and has nothing to prune.
clearCalls()
vi.setSystemTime(new Date(pruneAt.getTime() + 30_000))
await expect(refresh()).rejects.toThrow('ENOTFOUND')
expect(readPolicyNames(apiserver.api)).toEqual(
expect.arrayContaining(RUNTIME_EGRESS_POLICIES)
)
expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
// DNS recovers with the same set: the refresh finds the policies
// converged and keeps the resolved-at the prune preserved.
clearCalls()
vi.setSystemTime(new Date(pruneAt.getTime() + 60_000))
await expect(refresh()).resolves.toEqual({ conflicts: [], retryPending: false })
expect(resolve).toHaveBeenCalledTimes(5)
expect(readPolicyNames(apiserver.api)).toEqual(
expect.arrayContaining(RUNTIME_EGRESS_POLICIES)
)
expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
for (const name of RUNTIME_EGRESS_POLICIES) {
expect(liveAnnotations(name)?.[RESOLVED_AT]).toBe(SECOND_PASS_AT.toISOString())
expect(
logs.entries.filter(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
)
).toHaveLength(1)
}
} finally {
logs.restore()
}
})
describe('DNS rollover while the snippet runner policy is left unwritten', () => {
const COORD = 'test-wf-coord-to-wrc'
const RUNNER = 'test-wf-snippet-runner-egress'
const OLD_CIDR = '93.184.216.34/32'
const NEW_CIDR = '93.184.216.35/32'
const CURRENT_CIDRS = 'clerum.io/runtime-http-egress-current-cidrs'
const PREVIOUS_CIDRS = 'clerum.io/runtime-http-egress-previous-cidrs'
const REFRESH_INTERVAL_MS = 30_000
// 40 minutes of refreshes: eight times the 300 s overlap window.
const REFRESHES_AFTER_ROLLOVER = 80
/**
* Create both policies on the old CIDR, freeze the runner policy, move
* DNS to the new CIDR and refresh every 30 s. Returns the coordinator
* replaces seen during the loop; the mocks and `logs` then hold only one
* further refresh.
*/
async function rolloverWithFrozenRunner(
logs: ReturnType<typeof captureRunLaneNetworkPolicyLogs>,
freeze: (runner: k8s.V1NetworkPolicy) => void
) {
const apiserver = makeApiserverNetworkingApi()
let resolved = [OLD_CIDR]
const resolve = vi.fn(async () => resolved)
const reconciler = new WorkflowReconciler(
makeDeps({
networkingApi: apiserver.api as never,
resolveRuntimeHttpEgressCidrs: resolve,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
// A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
const spec: WorkflowRecipeSpec = {
...runtimeEgressSpec(),
coordinatorImage: 'registry.example/custom-coordinator:1',
}
const refresh = () =>
reconciler.refreshRuntimeHttpEgressNetworkPolicies(
'sandbox-recipes',
'test-wf',
'uid-123',
spec
)
const coordKey = apiserver.key('sandbox-recipes', COORD)
const runnerKey = apiserver.key('sandbox-recipes', RUNNER)
vi.useFakeTimers({ toFake: ['Date'] })
let now = FIRST_PASS_AT.getTime()
vi.setSystemTime(now)
await refresh()
for (const policyKey of [coordKey, runnerKey]) {
expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(
OLD_CIDR
)
}
freeze(apiserver.live.get(runnerKey)!)
resolved = [NEW_CIDR]
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
for (let pass = 0; pass < REFRESHES_AFTER_ROLLOVER; pass += 1) {
now += REFRESH_INTERVAL_MS
vi.setSystemTime(now)
await refresh()
}
const coordReplacesDuringLoop = apiserver.api.replaceNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg)
.filter(arg => arg.name === COORD)
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
now += REFRESH_INTERVAL_MS
vi.setSystemTime(now)
const last = await refresh()
return { ...apiserver, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last }
}
it('drops the old CIDR from the owned policy and stops replacing it when the sibling has a foreign owner', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =
await rolloverWithFrozenRunner(logs, runner => {
runner.metadata = {
...runner.metadata,
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
}
})
// Liveness: every pass resolved DNS; the last one read both policies
// by name for the egress state and again for the apply, and reached
// the ownership branch for the runner.
expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)
expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])
expect(last).toEqual({
conflicts: [{ policy: RUNNER, reason: 'owner-reference-mismatch' }],
retryPending: false,
})
expect(
logs.entries.some(
entry =>
entry.level === 'warn' &&
entry.reason === 'owner-reference-mismatch' &&
String(entry.msg).includes(RUNNER)
)
).toBe(true)
// The foreign policy keeps the old CIDR it was frozen with.
expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)
// One replace opens the overlap, one closes it once it expires.
expect(coordReplacesDuringLoop).toHaveLength(2)
const lastReplaced = coordReplacesDuringLoop.at(-1)!.body
expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])
expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()
expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)
).toHaveLength(0)
expect(
logs.entries.filter(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)
)
).toHaveLength(1)
} finally {
logs.restore()
}
})
it('drops the old CIDR from the owned policy and stops replacing it when the sibling is terminating', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =
await rolloverWithFrozenRunner(logs, runner => {
runner.metadata = {
...runner.metadata,
deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
}
})
expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)
expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])
expect(last).toEqual({ conflicts: [], retryPending: true })
expect(
logs.entries.some(
entry =>
entry.level === 'warn' &&
entry.reason === 'terminating' &&
String(entry.msg).includes(`NetworkPolicy "${RUNNER}"`)
)
).toBe(true)
expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)
expect(coordReplacesDuringLoop).toHaveLength(2)
const lastReplaced = coordReplacesDuringLoop.at(-1)!.body
expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])
expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()
expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)
).toHaveLength(0)
expect(
logs.entries.filter(entry =>
String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)
)
).toHaveLength(1)
} finally {
logs.restore()
}
})
})
})
it('reports a foreign-owned run-lane policy as a condition and keeps the run alive', async () => {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
const foreign = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))
expect(foreign).toBeDefined()
// Drift the spec too, so that without the ownership check the pass would PUT.
foreign!.spec = { ...foreign!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
foreign!.metadata = {
...foreign!.metadata,
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
}
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
const expectedNames = RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
expect(readPolicyNames(api)).toEqual(expectedNames)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message: expect.stringContaining('test-wf-coord-to-wrc'),
}),
])
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
).toHaveLength(0)
// A conflict is not transient: it waits for an operator, not a timer.
expect(second.networkPolicyRetryPending).toBeFalsy()
})
it('flags a retry for a run-lane policy that is being deleted and leaves it unwritten', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
expect(first.networkPolicyRetryPending).toBeFalsy()
const terminating = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))
expect(terminating).toBeDefined()
// Drift the spec too, so that without the deletion check the pass would PUT.
terminating!.spec = {
...terminating!.spec!,
podSelector: { matchLabels: { drifted: 'yes' } },
}
terminating!.metadata = {
...terminating!.metadata,
deletionTimestamp: new Date('2026-09-23T10:00:00Z'),
}
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
expect(second.networkPolicyRetryPending).toBe(true)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
// A policy being deleted is not a conflict: the pass reached the apply
// and found none, so the owned set holds only the RetryPending marker.
expect(second.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'RetryPending',
}),
])
expect(
api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
).toHaveLength(0)
// The retry warn carries the decision's reason, not a fixed phrase.
expect(
logs.entries.filter(
entry =>
entry.level === 'warn' &&
String(entry.msg).includes('test-wf-coord-to-wrc') &&
String(entry.msg).includes('retrying later')
)
).toEqual([expect.objectContaining({ reason: 'terminating' })])
} finally {
logs.restore()
}
})
it('flags a contended retry, without failing, after a concurrent writer wins all three replaces of a run-lane policy', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
const contendedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
const contended = live.get(contendedKey)
expect(contended).toBeDefined()
contended!.spec = { ...contended!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
const liveResourceVersion = contended!.metadata!.resourceVersion
// Another writer updates the policy just before each of our replaces, so
// every one carries a resourceVersion that is already stale.
const apiserverReplace = api.replaceNamespacedNetworkPolicy.getMockImplementation()!
let concurrentWrites = 0
api.replaceNamespacedNetworkPolicy.mockImplementation(async arg => {
if (arg.name === 'test-wf-coord-to-wrc') {
concurrentWrites += 1
const current = live.get(contendedKey)!
live.set(contendedKey, {
...current,
metadata: { ...current.metadata, resourceVersion: `concurrent-${concurrentWrites}` },
})
}
return apiserverReplace(arg)
})
api.readNamespacedNetworkPolicy.mockClear()
api.replaceNamespacedNetworkPolicy.mockClear()
logs.entries.length = 0
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(
api.readNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
)
).toHaveLength(3)
// Bounded: three writes, the same ceiling as the create-then-two-replaces
// apply this PR replaces.
expect(
api.replaceNamespacedNetworkPolicy.mock.calls
.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
.map(([arg]) => arg.body.metadata?.resourceVersion)
).toEqual([liveResourceVersion, 'concurrent-1', 'concurrent-2'])
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBe(true)
expect(
logs.entries.filter(
entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
)
).toEqual([
expect.objectContaining({
reason: 'contended',
msg: 'NetworkPolicy "test-wf-coord-to-wrc" is still contended after 3 write attempts; retrying later',
}),
])
} finally {
logs.restore()
}
})
describe('replace failures after the read', () => {
async function reconcileWithFailingReplace(
failReplace: (
arg: { name: string; namespace: string },
live: Map<string, k8s.V1NetworkPolicy>,
api: ReturnType<typeof makeApiserverNetworkingApi>['api']
) => Promise<never> | undefined,
logs?: { entries: Array<Record<string, unknown>> }
) {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
const drifted = apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
expect(drifted).toBeDefined()
// What the first pass stored: the desired policy as the apiserver keeps it.
const convergedSpec = structuredClone(drifted!.spec!)
drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
const apiserverReplace =
apiserver.api.replaceNamespacedNetworkPolicy.getMockImplementation()!
apiserver.api.replaceNamespacedNetworkPolicy.mockImplementation(
async arg => failReplace(arg, apiserver.live, apiserver.api) ?? apiserverReplace(arg)
)
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
apiserver.api.createNamespacedNetworkPolicy.mockClear()
// Only the second pass is under test; drop the first pass's "Created" lines.
logs?.entries.splice(0)
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
const contendedCalls = (mock: typeof apiserver.api.readNamespacedNetworkPolicy) =>
mock.mock.calls.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
return {
second,
apiserver,
convergedSpec,
reads: contendedCalls(apiserver.api.readNamespacedNetworkPolicy),
replaces: apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'test-wf-coord-to-wrc'
),
creates: apiserver.api.createNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.body.metadata?.name === 'test-wf-coord-to-wrc'
),
}
}
it('recreates the policy in the same pass when it is deleted between the read and the replace', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { second, apiserver, convergedSpec, reads, replaces, creates } =
await reconcileWithFailingReplace((arg, live) => {
if (arg.name !== 'test-wf-coord-to-wrc') return undefined
// Another actor deletes the policy after our read, so the
// apiserver double answers the PUT with 404.
expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
return undefined
}, logs)
// Read, 404 on the PUT, read again (absent), create.
expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
expect(creates).toHaveLength(1)
// The recreated object is the desired policy, nothing wider.
expect(
apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
).toEqual(convergedSpec)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
logs.entries
.filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
.map(entry => [entry.level, entry.msg])
).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
} finally {
logs.restore()
}
})
it('propagates a replace error that is neither a conflict nor a deletion', async () => {
const { second, reads, replaces } = await reconcileWithFailingReplace(arg => {
if (arg.name !== 'test-wf-coord-to-wrc') return undefined
return Promise.reject(
Object.assign(new Error('HTTP-Code: 422 Message: NetworkPolicy is invalid'), {
code: 422,
})
)
})
expect(reads).toHaveLength(1)
expect(replaces).toHaveLength(1)
expect(second.workflowPhase).toBe('failed')
expect(second.message).toContain('HTTP-Code: 422')
})
it('recreates the policy in the same pass when it is deleted after a replace conflict', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { second, apiserver, convergedSpec, reads, replaces, creates } =
await reconcileWithFailingReplace((arg, live) => {
if (arg.name !== 'test-wf-coord-to-wrc') return undefined
// Another actor deletes the policy and our PUT loses the race, so
// the re-read after the conflict finds nothing.
expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
return Promise.reject({ code: 409 })
}, logs)
expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
expect(creates).toHaveLength(1)
expect(
apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
).toEqual(convergedSpec)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
logs.entries
.filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
.map(entry => [entry.level, entry.msg])
).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
} finally {
logs.restore()
}
})
it('leaves an equal copy alone when another writer recreates the policy between our 404 and our create', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
let createRaces = 0
const { second, apiserver, convergedSpec, reads, replaces, creates } =
await reconcileWithFailingReplace((arg, live, api) => {
if (arg.name !== 'test-wf-coord-to-wrc') return undefined
expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
// Before our create lands, another writer stores the desired
// policy, so our POST answers 409 and the next read finds it.
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
api.createNamespacedNetworkPolicy.mockImplementationOnce(async create => {
createRaces += 1
await apiserverCreate(create)
return apiserverCreate(create)
})
return undefined
}, logs)
expect(createRaces).toBe(1)
// Read, 404 on the PUT, read (absent), create 409, read (equal).
expect(reads).toHaveLength(3)
expect(replaces).toHaveLength(1)
expect(creates).toHaveLength(1)
expect(
apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
).toEqual(convergedSpec)
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
logs.entries
.filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
.map(entry => [entry.level, entry.msg])
).toEqual([['info', 'Unchanged NetworkPolicy "test-wf-coord-to-wrc"; skipping update']])
} finally {
logs.restore()
}
})
// The decision is taken again on the object the re-read returns, not
// reused from the first read: a writer that wins the replace may also
// take the policy over or start deleting it.
function concurrentWriterOnFirstReplace(
rewrite: (current: k8s.V1NetworkPolicy) => k8s.V1NetworkPolicy
) {
let concurrentWrites = 0
return (
arg: { name: string; namespace: string },
live: Map<string, k8s.V1NetworkPolicy>
): Promise<never> | undefined => {
if (arg.name !== 'test-wf-coord-to-wrc' || concurrentWrites > 0) return undefined
concurrentWrites += 1
const liveKey = `${arg.namespace}/${arg.name}`
const current = live.get(liveKey)
expect(current).toBeDefined()
const rewritten = rewrite(current!)
live.set(liveKey, {
...rewritten,
metadata: { ...rewritten.metadata, resourceVersion: 'concurrent-1' },
})
return Promise.reject({ code: 409 })
}
}
it('reports a conflict, without a second replace, when the re-read after a replace conflict finds a foreign owner', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { second, reads, replaces } = await reconcileWithFailingReplace(
concurrentWriterOnFirstReplace(current => ({
...current,
metadata: {
...current.metadata,
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
},
}))
)
expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message:
'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
}),
])
// A conflict waits for an operator, not a timer.
expect(second.networkPolicyRetryPending).toBeFalsy()
expect(
logs.entries.filter(
entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
)
).toEqual([
expect.objectContaining({
reason: 'owner-reference-mismatch',
msg: 'NetworkPolicy "test-wf-coord-to-wrc" is not owned by this workflow; leaving it',
}),
])
} finally {
logs.restore()
}
})
it('flags a retry, without a second replace, when the re-read after a replace conflict finds the policy terminating', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { second, reads, replaces } = await reconcileWithFailingReplace(
concurrentWriterOnFirstReplace(current => ({
...current,
metadata: {
...current.metadata,
deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
},
}))
)
expect(reads).toHaveLength(2)
expect(replaces).toHaveLength(1)
expect(second.phase).not.toBe('failed')
expect(second.workflowPhase).not.toBe('failed')
expect(second.networkPolicyRetryPending).toBe(true)
// Being deleted is not a conflict: the owned set holds only the
// RetryPending marker.
expect(second.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'RetryPending',
}),
])
expect(
logs.entries.filter(
entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
)
).toEqual([
expect.objectContaining({
reason: 'terminating',
msg: 'NetworkPolicy "test-wf-coord-to-wrc" cannot be written yet; retrying later',
}),
])
} finally {
logs.restore()
}
})
})
describe('create conflicts, read errors and stale runtime state', () => {
const CONTENDED = 'test-wf-coord-to-wrc'
function callsFor<T extends { name?: string; body?: k8s.V1NetworkPolicy }>(mock: {
mock: { calls: Array<[T]> }
}): T[] {
return mock.mock.calls
.map(([arg]) => arg)
.filter(arg => (arg.name ?? arg.body?.metadata?.name) === CONTENDED)
}
it('replaces the winner of a create race with the winner resourceVersion', async () => {
const { api, live, key } = makeApiserverNetworkingApi()
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
let winnerResourceVersion: string | undefined
api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
if (arg.body.metadata?.name === CONTENDED) {
// Another writer creates a drifted copy between our 404 read and our POST.
const winner = await apiserverCreate({
namespace: arg.namespace,
body: {
...arg.body,
spec: { ...arg.body.spec!, podSelector: { matchLabels: { app: 'race-winner' } } },
},
})
winnerResourceVersion = winner.metadata?.resourceVersion
}
return apiserverCreate(arg)
})
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(result.workflowPhase).not.toBe('failed')
expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
const replaces = callsFor(api.replaceNamespacedNetworkPolicy)
expect(replaces).toHaveLength(1)
expect(winnerResourceVersion).toBeDefined()
expect(replaces[0]!.body.metadata?.resourceVersion).toBe(winnerResourceVersion)
expect(live.get(key('sandbox-recipes', CONTENDED))!.spec!.podSelector).not.toEqual({
matchLabels: { app: 'race-winner' },
})
})
it('creates again in the same pass when a create conflicts and the re-read finds nothing', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api, live, key } = makeApiserverNetworkingApi()
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
let conflicts = 0
api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
// The winner of the first race is deleted again before our re-read.
if (arg.body.metadata?.name === CONTENDED && conflicts === 0) {
conflicts += 1
throw { code: 409 }
}
return apiserverCreate(arg)
})
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(conflicts).toBe(1)
expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(2)
expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
expect(live.has(key('sandbox-recipes', CONTENDED))).toBe(true)
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
expect(result.networkPolicyRetryPending).toBeFalsy()
expect(
logs.entries
.filter(entry => String(entry.msg).includes(CONTENDED))
.map(entry => [entry.level, entry.msg])
).toEqual([['info', `Created NetworkPolicy "${CONTENDED}"`]])
} finally {
logs.restore()
}
})
it('flags a contended retry after three creates that all conflict, and writes no fourth time', async () => {
const logs = captureRunLaneNetworkPolicyLogs()
try {
const { api } = makeApiserverNetworkingApi()
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
// Every winner is deleted again before our re-read.
if (arg.body.metadata?.name === CONTENDED) throw { code: 409 }
return apiserverCreate(arg)
})
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(3)
expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(3)
expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
expect(result.networkPolicyRetryPending).toBe(true)
expect(
logs.entries.filter(
entry => entry.level === 'warn' && String(entry.msg).includes(CONTENDED)
)
).toEqual([
expect.objectContaining({
reason: 'contended',
msg: `NetworkPolicy "${CONTENDED}" is still contended after 3 write attempts; retrying later`,
}),
])
} finally {
logs.restore()
}
})
it('fails the run on a create error other than 409 and does not replace', async () => {
const { api } = makeApiserverNetworkingApi()
const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
if (arg.body.metadata?.name === CONTENDED) {
throw Object.assign(new Error('HTTP-Code: 500 Message: etcd unavailable'), {
code: 500,
})
}
return apiserverCreate(arg)
})
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)
expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
// A 500 is not a conflict: it leaves the apply and reaches the reconcile catch.
expect(result.message).toContain('HTTP-Code: 500')
})
it('fails the run on a read error other than 404 and writes nothing for that policy', async () => {
const { api } = makeApiserverNetworkingApi()
const apiserverRead = api.readNamespacedNetworkPolicy.getMockImplementation()!
api.readNamespacedNetworkPolicy.mockImplementation(async arg => {
if (arg.name === CONTENDED) {
throw Object.assign(new Error('HTTP-Code: 403 Message: forbidden'), { code: 403 })
}
return apiserverRead(arg)
})
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
// A 403 is not absence: creating over it would hide a permission defect.
expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)
expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(0)
expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('HTTP-Code: 403')
})
it('replaces a policy whose only drift is a stale runtime HTTP egress annotation', async () => {
const { api, live, key } = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
const STALE = 'clerum.io/runtime-http-egress-previous-cidrs'
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(first.workflowPhase).not.toBe('failed')
const stale = live.get(key('sandbox-recipes', CONTENDED))
expect(stale).toBeDefined()
expect(stale!.metadata?.annotations?.[STALE]).toBeUndefined()
stale!.metadata = {
...stale!.metadata,
annotations: { ...stale!.metadata?.annotations, [STALE]: '198.51.100.7/32' },
}
api.replaceNamespacedNetworkPolicy.mockClear()
const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
expect(second.workflowPhase).not.toBe('failed')
const replaces = callsFor(api.replaceNamespacedNetworkPolicy)
expect(replaces).toHaveLength(1)
expect(replaces[0]!.body.metadata?.annotations?.[STALE]).toBeUndefined()
expect(live.get(key('sandbox-recipes', CONTENDED))!.metadata?.annotations?.[STALE]).toBe(
undefined
)
})
})
// The recipe reconciler calls this from the running and active
// short-circuits, which never reach reconcile(), to retry a policy a
// previous pass left pending.
describe('retryRunLaneNetworkPolicies', () => {
const spec = () =>
makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
async function convergedRunLane() {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
expect(first.workflowPhase).not.toBe('failed')
expect([...apiserver.live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)
// reconcile() LISTs leftover run-lane policies and still DELETEs the
// legacy mcp-servers-egress-internet once (B3(c)). A zero delete count
// below means the retry skipped those paths, not that no prune exists.
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalled()
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.readNamespacedNetworkPolicy.mockClear()
apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
return { apiserver, reconciler }
}
it('recreates an absent run-lane policy and deletes nothing', async () => {
const { apiserver, reconciler } = await convergedRunLane()
const absentKey = apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
const convergedSpec = structuredClone(apiserver.live.get(absentKey)!.spec)
expect(apiserver.live.delete(absentKey)).toBe(true)
const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
expect(summary).toEqual({ conflicts: [], retryPending: false })
expect(readPolicyNames(apiserver.api)).toEqual(
RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
)
expect(
apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => arg.body.metadata?.name
)
).toEqual(['test-wf-coord-to-wrc'])
expect(apiserver.live.get(absentKey)!.spec).toEqual(convergedSpec)
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
it('writes nothing when every run-lane policy is live and equal', async () => {
const { apiserver, reconciler } = await convergedRunLane()
const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
expect(summary).toEqual({ conflicts: [], retryPending: false })
expect(readPolicyNames(apiserver.api)).toEqual(
RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
)
expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
it('still reports the retry, and writes nothing, while a policy is being deleted', async () => {
const { apiserver, reconciler } = await convergedRunLane()
const terminating = apiserver.live.get(
apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
)!
terminating.spec = {
...terminating.spec!,
podSelector: { matchLabels: { drifted: 'yes' } },
}
terminating.metadata = {
...terminating.metadata,
deletionTimestamp: new Date('2026-09-24T10:00:00Z'),
}
const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
expect(summary).toEqual({ conflicts: [], retryPending: true })
expect(readPolicyNames(apiserver.api)).toContain('test-wf-coord-to-wrc')
expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
// The retry runs only from the in-progress and active short-circuits,
// where the run's pods are live and may still use any lane the spec
// wanted when they were created. It applies what the current spec wants
// and prunes nothing; the next reconcile() pass prunes the rest.
describe('when the spec no longer wants a live lane', () => {
const SNIPPET_LANES = [
'test-wf-coord-to-snippet-runner',
'test-wf-coord-to-snippet-runner-ingress',
'test-wf-snippet-runner-egress',
]
const AGENT_LANE = 'test-wf-coord-to-mcp-host'
it('keeps it on a mid-run retry and still applies the lanes the spec wants', async () => {
const { apiserver, reconciler } = await convergedRunLane()
apiserver.api.listNamespacedNetworkPolicy.mockClear()
// An agent recipe with a run id wants the mcp-host lanes and none of
// the snippet lanes the converged snippet recipe created.
const summary = await reconciler.retryRunLaneNetworkPolicies(
'test-wf',
'uid-123',
makeSpec(),
'test-wf',
'run-1'
)
// Liveness witness: the retry built the agent lane and created it.
expect(
apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => arg.body.metadata?.name
)
).toContain(AGENT_LANE)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
for (const name of SNIPPET_LANES) {
expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(true)
}
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(summary).toEqual({ conflicts: [], retryPending: false })
})
it('prunes it on the next reconcile() pass', async () => {
const { apiserver, reconciler } = await convergedRunLane()
await reconciler.retryRunLaneNetworkPolicies(
'test-wf',
'uid-123',
makeSpec(),
'test-wf',
'run-1'
)
expect(
apiserver.live.has(apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
).toBe(true)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
'test-wf',
'run-1'
)
expect(result.workflowPhase).not.toBe('failed')
const deleted = apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(
([arg]) => arg.name
)
for (const name of SNIPPET_LANES) {
expect(deleted, name).toContain(name)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(false)
}
expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
})
})
// A reconcile() prune that failed publishes PrunePending with no retry
// flag. The retry does not prune and carries no prune fact, so the
// marker stays until the next reconcile() pass prunes the leftover.
describe('when a prune left a leftover behind', () => {
// A catalog member the snippet lane does not want and that no
// eligibility verdict governs.
const LEFTOVER = 'test-wf-coord-to-mcp-host'
const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'
const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'
type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']
function seedLeftover(apiserver: Converged) {
apiserver.live.set(apiserver.key('sandbox-recipes', LEFTOVER), {
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: {
name: LEFTOVER,
namespace: 'sandbox-recipes',
labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },
},
spec: { podSelector: {}, policyTypes: ['Egress'] },
})
}
// nit-3: the WRC suite stubs reconcile() with the real apply and the
// `reconcile` marker writer; this pins that the real reconcile()
// publishes exactly that writer's conditions for its own summary.
it('nit-3: publishes exactly the reconcile marker writer conditions for its own pass summary', async () => {
const { apiserver, reconciler } = await convergedRunLane()
seedLeftover(apiserver)
apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
code: 403,
message: 'forbidden',
})
const apply = vi.spyOn(
reconciler as unknown as {
applyWorkflowNetworkPolicies: (
...args: unknown[]
) => Promise<NetworkPolicyPassSummary>
},
'applyWorkflowNetworkPolicies'
)
const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
// Liveness witness: the pass ran the run-lane apply once, and its
// prune could not delete the leftover.
expect(apply).toHaveBeenCalledTimes(1)
const summary = await apply.mock.results[0]!.value
expect(summary).toEqual({
conflicts: [],
retryPending: false,
prune: 'pending',
legacy: 'removed',
})
const published = failed.networkPolicyOwnershipConditions ?? []
expect(published).toHaveLength(1)
expect(published).toEqual(
networkPolicyMarkerConditions(
{ kind: 'reconcile', summary },
undefined,
published[0]!.lastTransitionTime!
)
)
expect(failed.networkPolicyRetryPending).toBeUndefined()
})
it('keeps it and its PrunePending marker through the retry, and prunes it on the next reconcile()', async () => {
const { apiserver, reconciler } = await convergedRunLane()
seedLeftover(apiserver)
apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
code: 403,
message: 'forbidden',
})
const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
expect(failed.networkPolicyRetryPending).toBeFalsy()
const published = failed.networkPolicyOwnershipConditions ?? []
expect(
published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')
).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
apiserver.api.listNamespacedNetworkPolicy.mockClear()
apiserver.api.createNamespacedNetworkPolicy.mockClear()
apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
expect(
apiserver.live.delete(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
).toBe(true)
const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
// Liveness witness: the retry applied the lane the spec wants.
expect(
apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => arg.body.metadata?.name
)
).toEqual(['test-wf-coord-to-wrc'])
expect(retried).toEqual({ conflicts: [], retryPending: false })
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
const pruned = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
spec(),
{
conditions: published,
}
)
expect(pruned.networkPolicyRetryPending).toBeFalsy()
// The prune that landed clears the marker: the pass publishes its
// conditions and none of them is of that type any more.
expect(pruned.networkPolicyOwnershipConditions).toBeDefined()
expect(
(pruned.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowNetworkPoliciesConverged'
)
).toEqual([])
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'sandbox-recipes',
labelSelector: TWO_TERM_SELECTOR,
})
expect(
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
).toEqual([LEFTOVER])
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
})
it('R4-L6: publishes PruneUnevaluated, with no retry flag, from a reconcile() whose prune LIST failed', async () => {
const { apiserver, reconciler } = await convergedRunLane()
seedLeftover(apiserver)
const liveList = apiserver.api.listNamespacedNetworkPolicy.getMockImplementation()!
apiserver.api.listNamespacedNetworkPolicy.mockImplementation(
async (arg: { namespace: string; labelSelector?: string }) => {
if (arg.labelSelector === TWO_TERM_SELECTOR) throw { code: 500, message: 'boom' }
return liveList(arg)
}
)
apiserver.api.listNamespacedNetworkPolicy.mockClear()
const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
// Liveness witness: the prune issued its LIST, and it failed.
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: 'sandbox-recipes',
labelSelector: TWO_TERM_SELECTOR,
})
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
expect(failed.networkPolicyRetryPending).toBeFalsy()
expect(
(failed.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowNetworkPoliciesConverged'
)
).toMatchObject([{ status: 'False', reason: 'PruneUnevaluated' }])
// Witness: once the LIST answers, a DELETE that does not land is
// still PrunePending, and it replaces the unevaluated marker.
apiserver.api.listNamespacedNetworkPolicy.mockImplementation(liveList)
apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
code: 403,
message: 'forbidden',
})
const pending = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
spec(),
{ conditions: failed.networkPolicyOwnershipConditions ?? [] }
)
expect(apiserver.api.deleteNamespacedNetworkPolicy.mock.calls[0]?.[0].name).toBe(LEFTOVER)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
expect(pending.networkPolicyRetryPending).toBeFalsy()
expect(
(pending.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowNetworkPoliciesConverged'
)
).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
})
// The legacy mcp-servers internet policy is deleted by name, apart
// from the catalog prune: reconcile() deletes it, and the WRC's
// legacy-retry writer retries it (R4-L4). A failed delete is its own
// fact (R4-L3), DeletePending on its own condition, with no retry flag
// and the catalog prune converged, and the next reconcile() pass after
// the backoff window deletes it again.
const LEGACY = 'test-wf-mcp-servers-egress-internet'
const legacyDeletes = (apiserver: Converged) =>
apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === LEGACY
)
it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile() after its backoff window', async () => {
const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(t0)
try {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!
apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
async (arg: { name: string; namespace: string }) => {
if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {
throw { code: 403, message: 'forbidden' }
}
return liveDelete(arg)
}
)
const failed = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
spec()
)
expect(legacyDeletes(apiserver)).toHaveLength(1)
expect(failed.networkPolicyRetryPending).toBeFalsy()
const published = failed.networkPolicyOwnershipConditions ?? []
expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])
expect(
published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')
).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
vi.setSystemTime(t0 + 60_000)
const pruned = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
spec(),
{
conditions: published,
}
)
expect(legacyDeletes(apiserver)).toHaveLength(2)
expect(pruned.networkPolicyRetryPending).toBeFalsy()
expect(pruned.networkPolicyOwnershipConditions).toEqual([])
} finally {
vi.useRealTimers()
}
})
it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {
const apiserver = makeApiserverNetworkingApi()
const reconciler = new WorkflowReconciler(
makeDeps({ networkingApi: apiserver.api as never })
)
const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
// Liveness witness: the retry applied the lanes the spec wants.
expect(
apiserver.api.createNamespacedNetworkPolicy.mock.calls
.map(([arg]) => arg.body.metadata?.name)
.sort()
).toEqual(RUN_LANE_POLICY_NAMES.map(key => key.split('/')[1]).sort())
expect(retried).toEqual({ conflicts: [], retryPending: false })
expect(legacyDeletes(apiserver)).toEqual([])
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
// The retry computes its own verdict, and the verdict decides only
// which proxy it creates: the retry prunes nothing (see the per-path
// table on retryRunLaneNetworkPolicies), so a live proxy survives
// every verdict and one case covers them all. Both cases assert the
// create of a policy the spec wants as the liveness witness, so a
// negative assertion cannot pass because the retry never applied.
describe('with a fresh Codex/Grok verdict', () => {
const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'
const AGENT_LANE = 'test-wf-coord-to-mcp-host'
const CODEX_PROXY = 'test-wf-mcp-host-to-codex-proxy'
const projection = (overrides: Record<string, unknown>) => ({
targets: [],
eligibleTargets: [],
derivedScopes: [],
requiresCodexProxyEgress: false,
requiresGrokProxyEgress: false,
driftHashInput: '',
catalogContentHash: null,
catalogRevision: null,
connectionRevision: null,
eligibility: 'ineligible',
reason: 'static_only',
...overrides,
})
const REQUIRED = { eligibility: 'eligible', reason: 'granted' }
// The proxy policies as the factory emits them for this recipe, stored
// the way the apiserver returns them.
function seedProxies(apiserver: Converged) {
const config = makeConfig()
const proxies = buildWorkflowNetworkPolicies(
{
recipeName: 'test-wf',
sandboxNamespace: config.sandboxNamespace,
controlPlaneNamespace: config.controlPlaneNamespace,
mcpServerNamespace: config.mcpServerNamespace,
wrcPort: config.wrcPort,
mcpHostPort: config.mcpHostPort,
includeMcpHost: true,
includeCodexProxyEgress: true,
includeGrokProxyEgress: true,
},
[]
).filter(policy => [CODEX_PROXY, GROK_PROXY].includes(policy.metadata?.name ?? ''))
expect(proxies.map(policy => policy.metadata?.name).sort()).toEqual([
CODEX_PROXY,
GROK_PROXY,
])
for (const policy of proxies) {
apiserver.live.set(
apiserver.key('sandbox-recipes', policy.metadata!.name!),
asApiserverNetworkPolicy({
...policy,
metadata: { ...policy.metadata, namespace: 'sandbox-recipes' },
})
)
}
}
async function retryWith(
verdict: { codex: Record<string, unknown>; grok: Record<string, unknown> },
seed: boolean
) {
const { apiserver, reconciler } = await convergedRunLane()
apiserver.api.listNamespacedNetworkPolicy.mockClear()
if (seed) seedProxies(apiserver)
const verdictSpy = vi.spyOn(reconciler as never, 'codexVerdictFor').mockReturnValue({
projection: projection(verdict.codex),
grokProjection: projection(verdict.grok),
} as never)
try {
// An agent recipe with a run id needs mcp-host, the only consumer of
// the Codex/Grok proxy policies; the snippet spec above never wants them.
const summary = await reconciler.retryRunLaneNetworkPolicies(
'test-wf',
'uid-123',
makeSpec(),
'test-wf',
'run-1'
)
expect(verdictSpy).toHaveBeenCalledTimes(1)
return { apiserver, summary }
} finally {
verdictSpy.mockRestore()
}
}
const createdNames = (apiserver: Converged) =>
apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
([arg]) => arg.body.metadata?.name
)
// Decision: the retry never revokes a Codex/Grok proxy. It runs only
// while a run is in progress or active, and a step of the running run
// may still be calling the proxy. The retired verdict is the one under
// which reconcile() deletes the proxy, so it is the case pinned here.
it('never revokes a live proxy mid-run, whatever the verdict', async () => {
const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, true)
expect(createdNames(apiserver)).toContain(AGENT_LANE)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(true)
expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_PROXY))).toBe(true)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
// A kept proxy is not a pending retry, so the marker can clear.
expect(summary.retryPending).toBe(false)
})
// The verdict-dependent half: a required proxy is created (an allow)
// and a retired one is not, so a retry that ignores the verdict,
// creating both or neither, fails here.
it('creates only the proxy the fresh verdict requires', async () => {
const { apiserver, summary } = await retryWith(
{ codex: { ...REQUIRED, requiresCodexProxyEgress: true }, grok: {} },
false
)
expect(createdNames(apiserver)).toContain(AGENT_LANE)
expect(createdNames(apiserver)).toContain(CODEX_PROXY)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_PROXY))).toBe(true)
expect(createdNames(apiserver)).not.toContain(GROK_PROXY)
expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(false)
expect(summary.retryPending).toBe(false)
})
})
})
})
})
it('resolves snippet MCP transport workloads into the mounted runner config', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
workloads: [
{
id: 'mock-tools',
type: 'deployment',
image: 'clerum/mock-mcp-server:test',
port: 3000,
transport: { type: 'streamableHttp', path: '/mcp' },
},
],
steps: [
{
id: 'call-mcp',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 40, b: 2 })',
capabilities: {
mcp: {
servers: ['mock-tools'],
allowedTools: { include: ['mock-tools__add'] },
},
},
},
},
],
})
)
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.mcpServers).toHaveLength(1)
expect(config.mcpServers[0]).toMatchObject({
id: 'mock-tools',
endpoint: expect.stringContaining('.mcp-server.svc.cluster.local:3000/mcp'),
transport: 'streamableHttp',
})
expect(config.steps[0].run.capabilities.mcp.allowedTools.include).toEqual(['mock-tools__add'])
})
it('uses the runtime-safe MCP server label in snippet NetworkPolicy selectors for long child recipes', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
config: { ...makeConfig(), enableSnippetRuntime: true } as never,
})
)
const recipeName = 'manual-pr259-layer3a-hybrid-secret-pvc-5step-7f99549a'
const recipeUid = 'long-child-uid'
const spec = makeSpec({
agent: undefined,
mcpServers: [{ id: 'mock-tools' }],
workloads: [
{
id: 'mock-tools',
type: 'deployment',
image: 'clerum/mock-mcp-server:test',
port: 3000,
transport: { type: 'streamableHttp', path: '/mcp' },
},
],
steps: [
{
id: 'call-mcp',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 40, b: 2 })',
capabilities: {
mcp: {
servers: ['mock-tools'],
allowedTools: { include: ['mock-tools__add'] },
},
},
},
},
],
})
await reconciler.reconcile(recipeName, recipeUid, 'sandbox-recipes', spec)
const recipeRef = {
apiVersion: 'clerum.io/v1alpha1' as const,
kind: 'WorkflowRecipe' as const,
metadata: { name: recipeName, namespace: 'sandbox-recipes', uid: recipeUid },
spec,
}
const expectedLabel = resolveWorkloadMcpServerLabel(recipeRef, spec.workloads![0])
const rawInvalidLabel = `${recipeName}-mock-tools`
const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === `${recipeName}-snippet-runner-egress`
)?.[0].body
const selectedMcpServers = snippetEgress!.spec!.egress!.flatMap(rule =>
(rule.to ?? [])
.map(peer => peer.podSelector?.matchLabels?.['clerum.io/mcpserver'])
.filter(Boolean)
)
expect(rawInvalidLabel.length).toBeGreaterThan(63)
expect(expectedLabel.length).toBeLessThanOrEqual(63)
expect(selectedMcpServers).toContain(expectedLabel)
expect(selectedMcpServers).not.toContain(rawInvalidLabel)
})
it('resolves hybrid step MCP server IDs and step-level agent settings into coordinator config', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
mcpServers: [
{
id: 'mock-tools',
endpoint: 'http://mock-tools.mcp-server.svc.cluster.local:3000/mcp',
},
],
steps: [
{ id: 'prepare', run: snippetRun() },
{
id: 'calculate',
dependsOn: ['prepare'],
instruction: 'Use the add tool.',
agent: { provider: 'zai', model: 'glm-4.7' },
mcpServers: ['mock-tools'],
allowedTools: { include: ['mock-tools__add'] },
maxIterations: 6,
},
],
})
)
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.agent).toBeUndefined()
expect(config.steps[1]).toMatchObject({
id: 'calculate',
agent: { provider: 'zai', model: 'glm-4.7' },
mcpServers: [
{
name: 'mock-tools',
url: 'http://mock-tools.mcp-server.svc.cluster.local:3000/mcp',
},
],
allowedTools: { include: ['mock-tools__add'] },
maxIterations: 6,
})
})
it('creates coordinator and mcp-host Pods via coreApi', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(deps.coreApi.createNamespacedPod).toHaveBeenCalled()
})
it('creates a short mcp-host route alias Service for provider callback routing', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
const createdServiceNames = vi
.mocked(deps.coreApi.createNamespacedService)
.mock.calls.map(([arg]) => arg.body?.metadata?.name)
expect(createdServiceNames).toContain(
buildMcpHostRouteAliasServiceName('test-wf', 'sandbox-recipes')
)
})
describe('headless Services are read before they are written', () => {
const namespace = 'sandbox-recipes'
const mcpHostName = buildMcpHostServiceName('test-wf')
const aliasName = buildMcpHostRouteAliasServiceName('test-wf', namespace)
const readerName = buildArtifactReaderServiceName('test-wf')
const snippetRunnerName = buildSnippetRunnerServiceName('test-wf')
const reconcileTriggeredRun = () =>
new WorkflowReconciler(deps).reconcile(
'test-wf',
'uid-123',
namespace,
makeSpec(),
undefined,
undefined,
undefined,
runId
)
it('sends neither POST nor PUT when the live mcp-host and route alias Services match', async () => {
serveLiveServices(deps, [
asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
])
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
name: mcpHostName,
namespace,
})
expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
name: aliasName,
namespace,
})
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('reads, then creates the mcp-host and route alias Services exactly once when they are absent', async () => {
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
expect(serviceReadsOf(deps, aliasName)).toBe(1)
expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {
const desired = buildMcpHostHeadlessService('test-wf', namespace)
// clusterIP differs from the desired 'None' so the assertion below can
// only pass if the replace carries the live (immutable) value over.
const drifted = asLiveService(
{
...desired,
spec: {
...desired.spec,
clusterIP: '10.96.0.42',
ports: desired.spec?.ports?.slice(0, 1),
},
},
'rv-7'
)
serveLiveServices(deps, [
drifted,
asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
])
await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
expect(replaceCalls).toHaveLength(1)
const [{ name, namespace: replacedIn, body }] = replaceCalls[0]
expect(name).toBe(mcpHostName)
expect(replacedIn).toBe(namespace)
expect(body.metadata?.resourceVersion).toBe('rv-7')
expect(body.spec?.clusterIP).toBe('10.96.0.42')
expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
})
describe('when the drift replace conflicts with a concurrent write', () => {
const driftedMcpHost = (resourceVersion: string) => {
const desired = buildMcpHostHeadlessService('test-wf', namespace)
return asLiveService(
{ ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
resourceVersion
)
}
/** Answer successive mcp-host reads in order; a read past the list fails loudly. */
const serveMcpHostReads = (...responses: Array<LiveService | 404>) => {
let reads = 0
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name !== mcpHostName) throw { code: 404 }
const response = responses[reads]
reads += 1
if (response === undefined) throw new Error(`unexpected read #${reads} of ${mcpHostName}`)
if (response === 404) throw { code: 404 }
return structuredClone(response)
})
}
const conflictOnFirstReplaces = (conflicts: number) => {
let replaces = 0
vi.mocked(deps.coreApi.replaceNamespacedService).mockImplementation(async ({ body }) => {
replaces += 1
if (replaces <= conflicts) throw { code: 409 }
return body
})
}
const replaceCalls = () => vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
it('re-reads and retries the replace once with the fresh resourceVersion', async () => {
serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
conflictOnFirstReplaces(1)
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([
'rv-7',
'rv-8',
])
expect(replaceCalls()[1][0].body.spec?.ports?.map(port => port.name)).toEqual([
'http',
'plugin-sdk',
])
})
it('stops without a second write when the re-read Service already matches', async () => {
serveMcpHostReads(
driftedMcpHost('rv-7'),
asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-8')
)
conflictOnFirstReplaces(1)
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(replaceCalls()).toHaveLength(1)
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
})
it('asks for a fresh pass, without failing, when the Service vanishes after the conflict', async () => {
serveMcpHostReads(driftedMcpHost('rv-7'), 404)
conflictOnFirstReplaces(1)
const result = await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(replaceCalls()).toHaveLength(1)
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
expect(result.workflowPhase).not.toBe('failed')
expect(result.phase).not.toBe('failed')
expect(result.skipStatusPatch).toBe(true)
expect(result.message).toContain(
`Headless Service "${mcpHostName}" disappeared after a replace conflict`
)
})
it('asks for a fresh pass, without failing, when the replace conflicts on both attempts', async () => {
serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
conflictOnFirstReplaces(2)
const result = await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([
'rv-7',
'rv-8',
])
expect(result.workflowPhase).not.toBe('failed')
expect(result.phase).not.toBe('failed')
expect(result.skipStatusPatch).toBe(true)
expect(result.message).toContain(
`Headless Service "${mcpHostName}" replace conflicted on both attempts`
)
})
it('propagates a replace error other than a conflict without retrying', async () => {
serveMcpHostReads(driftedMcpHost('rv-7'))
vi.mocked(deps.coreApi.replaceNamespacedService).mockRejectedValue(
Object.assign(new Error('HTTP-Code: 422 Message: Service is invalid'), { code: 422 })
)
const result = await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
expect(replaceCalls()).toHaveLength(1)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('HTTP-Code: 422')
})
})
it('leaves a drifted route alias Service untouched because it is create-only', async () => {
const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)
serveLiveServices(deps, [
asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
asLiveService(
{
...desiredAlias,
spec: { ...desiredAlias.spec, ports: desiredAlias.spec?.ports?.slice(0, 1) },
},
'rv-2'
),
])
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, aliasName)).toBe(1)
expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {
const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')
let mcpHostReads = 0
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name !== mcpHostName) throw { code: 404 }
mcpHostReads += 1
if (mcpHostReads === 1) throw { code: 404 }
return structuredClone(live)
})
vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
if (body.metadata?.name === mcpHostName) throw { code: 409 }
return body
})
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('repairs a drifted mcp-host Service that another writer created after the read', async () => {
const desired = buildMcpHostHeadlessService('test-wf', namespace)
const racedDrifted = asLiveService(
{ ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
'rv-11'
)
let mcpHostReads = 0
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name !== mcpHostName) throw { code: 404 }
mcpHostReads += 1
if (mcpHostReads === 1) throw { code: 404 }
return structuredClone(racedDrifted)
})
vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
if (body.metadata?.name === mcpHostName) throw { code: 409 }
return body
})
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
expect(replaceCalls).toHaveLength(1)
const [{ name, body }] = replaceCalls[0]
expect(name).toBe(mcpHostName)
expect(body.metadata?.resourceVersion).toBe('rv-11')
expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
})
it('asks for a fresh pass, without failing, when the mcp-host Service vanishes after a create conflict', async () => {
vi.mocked(deps.coreApi.readNamespacedService).mockRejectedValue({ code: 404 })
vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
if (body.metadata?.name === mcpHostName) throw { code: 409 }
return body
})
const result = await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
expect(result.workflowPhase).not.toBe('failed')
expect(result.phase).not.toBe('failed')
expect(result.skipStatusPatch).toBe(true)
expect(result.message).toContain(
`Headless Service "${mcpHostName}" disappeared after create conflict`
)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {
const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
code: 403,
})
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name === mcpHostName) throw forbidden
throw { code: 404 }
})
const result = await reconcileTriggeredRun()
expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('HTTP-Code: 403')
expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
})
it('propagates a non-404 read error instead of creating a create-only Service blind', async () => {
const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
code: 403,
})
vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
if (name === aliasName) throw forbidden
throw { code: 404 }
})
const result = await reconcileTriggeredRun()
// The mcp-host Service is handled first and was created, so the pass reached the alias.
expect(createdServiceNamesOf(deps)).toContain(mcpHostName)
expect(serviceReadsOf(deps, aliasName)).toBe(1)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('HTTP-Code: 403')
expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
})
it('converges when another writer creates a create-only Service between the read and the POST', async () => {
vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
if (body.metadata?.name === aliasName) throw { code: 409 }
return body
})
const result = await reconcileTriggeredRun()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, aliasName)).toBe(1)
expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
describe('snippet-runner and artifact-reader Services', () => {
const reconcileSnippetWorkflow = () => {
vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({
metadata: { labels: { 'clerum.io/shared': 'true' } },
data: { key: 'dmFsdWU=' },
})
return new WorkflowReconciler(deps).reconcile(
'test-wf',
'uid-123',
namespace,
makeSpec({
agent: undefined,
workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
steps: [
{
id: 'snippet',
run: {
type: 'snippet',
language: 'typescript',
code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
capabilities: {
secrets: [
{ alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } },
],
postgres: { access: 'read', workloads: ['postgres'] },
},
},
},
],
})
)
}
beforeEach(() => {
deps = makeDeps({
config: {
...makeConfig(),
enableSnippetRuntime: true,
snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
} as never,
})
})
it('sends no POST for snippet-runner and artifact-reader Services that already exist', async () => {
serveLiveServices(deps, [
asLiveService(buildArtifactReaderHeadlessService('test-wf', namespace), 'rv-3'),
asLiveService(buildSnippetRunnerHeadlessService('test-wf', namespace), 'rv-4'),
])
const result = await reconcileSnippetWorkflow()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, readerName)).toBe(1)
expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
expect(createdServiceNamesOf(deps)).not.toContain(readerName)
expect(createdServiceNamesOf(deps)).not.toContain(snippetRunnerName)
expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
})
it('reads, then creates snippet-runner and artifact-reader Services when they are absent', async () => {
const result = await reconcileSnippetWorkflow()
expect(result.workflowPhase).not.toBe('failed')
expect(serviceReadsOf(deps, readerName)).toBe(1)
expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
expect(createdServiceNamesOf(deps).filter(name => name === readerName)).toHaveLength(1)
expect(createdServiceNamesOf(deps).filter(name => name === snippetRunnerName)).toHaveLength(
1
)
})
})
})
it('applies runtime NetworkPolicies before creating runtime Pods', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
const firstPolicyCall = vi.mocked(deps.networkingApi.createNamespacedNetworkPolicy).mock
.invocationCallOrder[0]
const runtimePodCall = vi
.mocked(deps.coreApi.createNamespacedPod)
.mock.calls.findIndex(
([arg]) => arg.body?.metadata?.name !== 'test-wf-workflow-output-anchor'
)
expect(runtimePodCall).toBeGreaterThanOrEqual(0)
const firstRuntimePodCall = vi.mocked(deps.coreApi.createNamespacedPod).mock
.invocationCallOrder[runtimePodCall]
expect(firstPolicyCall).toBeLessThan(firstRuntimePodCall)
})
it('registers an agentic parent workflow without runtime pods until a run id exists', async () => {
const coreApi = makeCoreApi(false, { workflowOutputAnchorPhase: null }) as ReturnType<
typeof makeCoreApi
>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({ coreApi: coreApi as never, networkingApi: networkingApi as never })
)
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(result).toMatchObject({
phase: 'active',
message: 'Workflow trigger infrastructure registered (workflow-agentic)',
clearWorkflowExecution: true,
})
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toEqual(['test-wf-workflow-output-anchor'])
expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
})
it('threads workflow run id into the coordinator pod env', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
const pod = getCreatedCoordinatorPod(coreApi)
const env = pod.spec!.containers![0].env!.find(
(e: { name: string }) => e.name === 'CLERUM_WORKFLOW_RUN_ID'
)
expect(env!.value).toBe(runId)
})
it('maps admin workflow actors to usage-only admin-ui user keys', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId,
'control-plane-admin-ui',
'11111111-1111-4111-8111-111111111111',
'admin'
)
const pod = getCreatedCoordinatorPod(coreApi)
const env = pod.spec!.containers![0].env!
expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_RUN_ID')!.value).toBe(
runId
)
expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_TEAM_ID')!.value).toBe(
'control-plane-admin-ui'
)
expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_USER_ID')!.value).toBe(
'admin-ui/11111111-1111-4111-8111-111111111111'
)
})
it('accepts pure snippet workflows and skips the mcp-host pod', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'render', run: snippetRun() }],
})
)
expect(result.phase).toBe('deploying')
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-coordinator')
expect(createdPodNames).not.toContain('test-wf-mcp-host')
expect(result.message).not.toContain('not yet implemented')
})
it('rejects custom coordinator images when the feature flag is disabled', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('custom coordinator images are disabled')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('runs pure custom coordinator workflows without mcp-host and with implicit platform output', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
})
)
expect(result.message).toContain('workflow-custom')
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-coordinator')
expect(createdPodNames).toContain('test-wf-artifact-reader')
expect(createdPodNames).not.toContain('test-wf-mcp-host')
const coordPod = getCreatedCoordinatorPod(coreApi)
const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
expect(coordPod.spec!.containers![0].image).toBe('clerum/workflow-custom-sdk-e2e:test')
expect(artifactReaderPod.spec!.containers![0].image).toBe('clerum/workflow-recipes:test')
const artifactReaderOutputMount = artifactReaderPod.spec!.containers![0].volumeMounts!.find(
(mount: { mountPath: string }) => mount.mountPath === '/output'
)
expect(artifactReaderOutputMount).toMatchObject({
name: 'recipe-output',
mountPath: '/output',
subPath: 'workflow-output/test-wf',
})
expect(artifactReaderOutputMount).not.toHaveProperty('readOnly')
expect(
artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expect(envNames).not.toContain('MCP_HOST_TOKEN')
expect(envNames).not.toContain('MCP_HOST_RUNTIME_ACCESS_TOKEN')
expect(coordPod.spec!.enableServiceLinks).toBe(false)
expect(coordPod.spec!.hostNetwork).toBe(false)
expect(coordPod.spec!.hostPID).toBe(false)
expect(coordPod.spec!.hostIPC).toBe(false)
expect(coordPod.spec!.activeDeadlineSeconds).toBe(3300)
expect(
coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'tmp')!.emptyDir
).toEqual({
sizeLimit: '64Mi',
})
expect(
coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expect(
coordPod.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)!.subPath
).toBe('workflow-output/test-wf')
expectWorkflowOutputAnchorAffinity(coordPod)
expectWorkflowOutputAnchorAffinity(artifactReaderPod)
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
const coordinatorSecret = coreApi.createNamespacedSecret.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'wf-test-wf-coordinator-token'
)?.[0].body
expect(coordinatorSecret!.data!['mcp-host-token']).toBeUndefined()
expect(Buffer.from(coordinatorSecret!.data!['wrc-token'], 'base64').toString()).toBe(
'custom-coord-wrc-token'
)
})
it('honors explicit custom coordinator pvc output while keeping the mount run scoped', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
output: { destination: 'pvc' },
}),
undefined,
undefined,
undefined,
runId
)
const coordPod = getCreatedCoordinatorPod(coreApi)
const outputMount = coordPod.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)
expect(outputMount).toMatchObject({
name: 'recipe-output',
mountPath: '/output',
subPath: `workflow-output/test-wf/${runId}`,
})
expect(
coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
})
it('propagates runtimeEgress and grants declared public HTTP egress to custom coordinators', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
runtimeEgress: { http: { allowedHosts: ['example.com'] } },
steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
output: { destination: 'pvc' },
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['example.com'])
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.runtimeEgress.http.allowedHosts).toEqual(['example.com'])
const coordPolicy = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-coord-to-wrc'
)?.[0].body
expect(coordPolicy).toBeDefined()
expect(coordPolicy!.metadata!.labels!['clerum.io/egress-class']).toBe('exact-host')
expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(true)
expect(publicHttpEgressCidrs(coordPolicy!)).toEqual(['93.184.216.34/32'])
})
it('grants explicit public-web egress to custom coordinators without DNS resolution', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const resolveRuntimeHttpEgressCidrs = vi.fn()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
resolveRuntimeHttpEgressCidrs,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
runtimeEgress: { http: { egressClass: 'public-web' } },
steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
output: { destination: 'pvc' },
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
const coordPolicy = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-coord-to-wrc'
)?.[0].body
expect(coordPolicy).toBeDefined()
expect(coordPolicy!.metadata!.labels!['clerum.io/egress-class']).toBe('public-web')
expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(true)
expect(publicHttpEgressCidrs(coordPolicy!)).toEqual(['0.0.0.0/0'])
})
it('keeps artifact-reader available for broker-backed custom coordinator output artifacts', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
output: { destination: 'pvc' },
steps: [
{ id: 'prepare' },
{
id: 'broker-review',
dependsOn: ['prepare'],
instruction: 'Use the broker',
agent: { provider: 'zai', model: 'glm-4.7' },
mcpServers: ['mock-tools'],
},
{ id: 'emit', dependsOn: ['broker-review'] },
],
}),
undefined,
undefined,
undefined,
runId
)
expect(result.message).toContain('workflow-custom')
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-coordinator')
expect(createdPodNames).toContain('test-wf-mcp-host')
expect(createdPodNames).toContain('test-wf-artifact-reader')
expect(getCreatedArtifactReaderPod(coreApi).spec!.containers![0].volumeMounts).toContainEqual(
expect.objectContaining({
name: 'recipe-output',
mountPath: '/output',
})
)
expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCoordinatorToMcpHostToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
})
it('persists resolved inputs into the mounted custom coordinator config', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const resolvedInputs = {
requestId: 'ui-custom-123',
approvalThreshold: 1000,
scenario: 'desktop-custom-coordinator-trigger',
}
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
inputContract: {
properties: {
requestId: { type: 'string', default: 'ui-custom-default' },
approvalThreshold: { type: 'number', default: 5000 },
scenario: { type: 'string', default: 'default-scenario' },
},
},
steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
output: { destination: 'pvc' },
}),
undefined,
resolvedInputs
)
const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
)?.[0].body
const config = JSON.parse(configMap!.data!['config.json'])
expect(config.inputs).toEqual(resolvedInputs)
expect(config.inputContract.properties.requestId.default).toBe('ui-custom-default')
})
it('rejects custom coordinator timeout budgets above the active deadline cap', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [
{ id: 'prepare', timeoutSeconds: 2400 },
{ id: 'emit', dependsOn: ['prepare'], timeoutSeconds: 2400 },
],
output: { destination: 'pvc' },
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('maximum active deadline')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('runs broker-backed custom coordinator workflows with mcp-host credentials only when needed', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: { provider: 'openai', model: 'gpt-4o-mini' },
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [
{ id: 'prepare' },
{ id: 'review', dependsOn: ['prepare'], instruction: 'Review the prepared result.' },
],
output: { destination: 'pvc' },
}),
undefined,
undefined,
undefined,
runId
)
expect(result.message).toContain('workflow-custom')
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-coordinator')
expect(createdPodNames).toContain('test-wf-mcp-host')
const coordPod = getCreatedCoordinatorPod(coreApi)
const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
expect(envNames).toContain('CLERUM_MCPHOST_URL')
expect(envNames).toContain('MCP_HOST_TOKEN_FILE')
expect(envNames).not.toContain('MCP_HOST_TOKEN')
for (const forbiddenEnv of [
'MCP_HOST_RUNTIME_ACCESS_TOKEN',
'MCP_HOST_RUNTIME_REFRESH_TOKEN',
'MCP_HOST_WORKFLOW_CONTROL_TOKEN',
'MCP_HOST_GATEWAY_URL',
'OPENAI_API_KEY',
'ZAI_API_KEY',
'BAILIAN_API_KEY',
'CLAUDE_API_KEY',
'CLERUM_MODEL_API_KEY',
]) {
expect(envNames).not.toContain(forbiddenEnv)
}
expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCoordinatorToMcpHostToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
})
it('keeps broker-backed custom workflows without explicit pvc output on WRC-managed output', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: { provider: 'openai', model: 'gpt-4o-mini' },
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [
{ id: 'prepare' },
{ id: 'review', dependsOn: ['prepare'], instruction: 'Review the prepared result.' },
],
}),
undefined,
undefined,
undefined,
runId
)
expect(result.message).toContain('workflow-custom')
const mcpHostPod = getCreatedMcpHostPod(coreApi)
const outputMount = mcpHostPod.spec!.containers![0].volumeMounts!.find(
(m: { mountPath: string }) => m.mountPath === '/output'
)
expect(outputMount!.subPath).toBe(`workflow-output/test-wf/${runId}`)
expect(
mcpHostPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
.persistentVolumeClaim!.claimName
).toBe('test-wf-workflow-output')
expectWorkflowOutputAnchorAffinity(mcpHostPod)
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
})
it('keeps existing custom coordinator token Secrets reduced on later reconciles', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
return {}
})
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
'base64'
),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
output: { destination: 'pvc' },
})
)
expect(result.message).toContain('workflow-custom')
expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledTimes(1)
expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
})
it('prunes stale mcp-host tokens from existing pure custom coordinator Secrets', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
return {}
})
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'mcp-host-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
'base64'
),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
output: { destination: 'pvc' },
})
)
expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
body: {
data: {
'mcp-host-token': null,
},
},
}),
expect.any(Object)
)
})
it('rotates an existing coordinator WRC token when a custom coordinator needs the custom subject', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
return {}
})
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'coordinator' })).toString(
'base64'
),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
output: { destination: 'pvc' },
})
)
expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCoordinatorToWrcToken).not.toHaveBeenCalled()
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
body: {
data: {
'wrc-token': Buffer.from('custom-coord-wrc-token').toString('base64'),
},
},
}),
expect.any(Object)
)
})
it('rotates an existing custom coordinator WRC token back to the builtin coordinator subject', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
return {}
})
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'mcp-host-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
'base64'
),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
)
await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCustomCoordinatorToWrcToken).not.toHaveBeenCalled()
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
body: {
data: {
'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
},
},
}),
expect.any(Object)
)
})
it('refreshes coordinator runtime credentials during steady-state snippet workflows', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'wrc-token': Buffer.from(
makeJwtWithExp(30, { sub: 'coordinator', recipeName: 'test-wf' })
).toString('base64'),
'snippet-runner-token': Buffer.from('snippet-runner-token').toString('base64'),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
)
await reconciler.ensureCoordinatorRuntimeCredentials(
'sandbox-recipes',
'test-wf',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
})
)
expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
namespace: 'sandbox-recipes',
body: {
data: {
'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
},
},
}),
expect.any(Object)
)
})
it('refreshes coordinator GFS access when declared publish scopes change', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'gfs-token': makeEncodedGfsAccess(3600, 'test-wf', ['gfs.read']),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureCoordinatorRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({
gfs: {
publishTargets: [{ drive: 'main', target: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }],
},
})
)
expect(mintRecipeHostGfsToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', {
scopes: ['gfs.read', 'gfs.write'],
})
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
namespace: 'recipe-runtime-ns',
body: {
data: expect.objectContaining({
'gfs-token': expect.any(String),
}),
},
}),
expect.any(Object)
)
})
it('refreshes short-TTL coordinator tokens early enough for Secret volume projection', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const tokenFactory = makeTokenFactory()
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-coordinator-token') {
return {
data: {
'wrc-token': Buffer.from(
makeJwtWithExp(60, { sub: 'coordinator', recipeName: 'test-wf' })
).toString('base64'),
'snippet-runner-token': Buffer.from('snippet-runner-token').toString('base64'),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
tokenFactory: tokenFactory as never,
config: {
...makeConfig(),
runtimeTokenTtlSeconds: 90,
runtimeTokenRefreshBeforeSeconds: 45,
} as never,
})
)
await reconciler.ensureCoordinatorRuntimeCredentials(
'sandbox-recipes',
'test-wf',
makeSpec({
agent: undefined,
steps: [{ id: 'snippet', run: snippetRun() }],
})
)
expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-coordinator-token',
namespace: 'sandbox-recipes',
body: {
data: {
'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
},
},
}),
expect.any(Object)
)
})
it('rejects coordinator image changes after the runtime pod already exists', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'test-wf-coordinator') {
return {
status: { phase: 'Running' },
spec: { containers: [{ image: 'clerum/coordinator:test' }] },
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('coordinator image is immutable')
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('allows built-in coordinator pods to finish after platform image config changes', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'test-wf-coordinator') {
return {
metadata: { labels: { 'clerum.io/coordinator-tier': 'builtin' } },
status: { phase: 'Running' },
spec: { containers: [{ image: 'clerum/coordinator:old' }] },
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
coordinatorImage: 'clerum/coordinator:new',
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'render', run: snippetRun() }],
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(result.message).not.toContain('coordinator image is immutable')
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).not.toContain('test-wf-coordinator')
})
it('rejects removing a custom coordinator image after the custom runtime pod already exists', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'test-wf-coordinator') {
return {
metadata: { labels: { 'clerum.io/coordinator-tier': 'custom' } },
status: { phase: 'Running' },
spec: { containers: [{ image: 'clerum/workflow-custom-sdk-e2e:test' }] },
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'render', run: snippetRun() }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('coordinator tier is immutable')
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects disallowed custom coordinator image prefixes before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'docker.io/unknown/custom:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('not allowed')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects adjacent custom coordinator image prefix bypasses', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/workflow'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'ghcr.io/your-org/workflow-evil:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('not allowed')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects custom coordinator image path traversal segments before prefix checks', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'ghcr.io/your-org/../../attacker/workflow:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('custom coordinator image reference is invalid')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects custom coordinator images when the allowlist is empty', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: [],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('allowlist is empty')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it.each([
{
caseName: 'slash boundary',
prefixes: ['ghcr.io/your-org'],
image: 'ghcr.io/your-org/workflow-custom:test',
},
{
caseName: 'tag boundary',
prefixes: ['clerum/workflow-custom-sdk-e2e'],
image: 'clerum/workflow-custom-sdk-e2e:test',
},
{
caseName: 'digest boundary',
prefixes: ['clerum/workflow-custom-sdk-e2e'],
image: `clerum/workflow-custom-sdk-e2e@sha256:${'a'.repeat(64)}`,
},
{
caseName: 'prefix ending with explicit tag delimiter',
prefixes: ['clerum/workflow-custom-sdk-e2e:'],
image: 'clerum/workflow-custom-sdk-e2e:test',
},
])('accepts custom coordinator allowlist $caseName', async ({ prefixes, image }) => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: prefixes,
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: image,
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).not.toBe('failed')
expect(coreApi.createNamespacedPod).toHaveBeenCalled()
})
it('rejects latest custom coordinator images', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:latest',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('must not use :latest')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects latest custom coordinator images even when a digest is present', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: `clerum/workflow-custom-sdk-e2e:latest@sha256:${'a'.repeat(64)}`,
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('must not use :latest')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects malformed custom coordinator digests', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test@sha256:nothex',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('valid sha256 digest')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects uppercase custom coordinator sha256 digests', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/'],
requireCoordinatorImageDigest: true,
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: `clerum/workflow-custom-sdk-e2e:test@sha256:${'A'.repeat(64)}`,
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('valid sha256 digest')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('enforces custom coordinator digest policy when configured', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/'],
requireCoordinatorImageDigest: true,
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [{ id: 'prepare' }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('valid sha256 digest')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('accepts valid custom coordinator digest refs when digest policy is configured', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e'],
requireCoordinatorImageDigest: true,
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: `clerum/workflow-custom-sdk-e2e:test@sha256:${'a'.repeat(64)}`,
steps: [{ id: 'prepare' }],
})
)
expect(result.message).toContain('workflow-custom')
expect(coreApi.createNamespacedPod).toHaveBeenCalled()
})
it('fails custom approval steps without a broker agent before creating runtime resources', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
enableCustomCoordinatorImage: true,
allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
} as never,
})
)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
steps: [
{
id: 'approval',
requiresApproval: {
target: { userId: 'operator' },
message: 'approve custom step',
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('requires an agent configuration for approval')
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('classifies all-run workflows as snippet workflows even when a default agent is present', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
steps: [{ id: 'render', run: snippetRun() }],
})
)
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(result.message).toContain('workflow-snippet')
expect(createdPodNames).toContain('test-wf-coordinator')
expect(createdPodNames).not.toContain('test-wf-mcp-host')
})
it('rejects run steps that also configure a step agent', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'ambiguous',
run: snippetRun(),
agent: { provider: 'zai', model: 'glm-4.7' },
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('cannot configure an agent when run is set')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects snippet run steps that require approval before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'gated',
run: snippetRun(),
requiresApproval: {
target: { userId: 'operator' },
message: 'approve snippet step',
},
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('cannot require approval')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('skips runtime token Secret provisioning for pure snippet workflows', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec({
agent: undefined,
steps: [{ id: 'render', run: snippetRun() }],
})
)
expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
})
it('does not mount mcp-host credentials on snippet coordinator pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'render', run: snippetRun() }],
})
)
const coordPod = getCreatedCoordinatorPod(coreApi)
const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
expect(envNames).not.toContain('MCP_HOST_ENDPOINT')
expect(envNames).not.toContain('MCP_HOST_TOKEN')
expect(envNames).not.toContain('MCP_HOST_RUNTIME_ACCESS_TOKEN')
})
it('mounts the parent output PVC on snippet coordinator pods when workflow output is enabled', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'pdf',
run: snippetRun(),
},
],
}),
undefined,
undefined,
undefined,
runId
)
const coordPod = getCreatedCoordinatorPod(coreApi)
const outputMount = coordPod.spec!.containers![0].volumeMounts!.find(
(mount: { mountPath: string }) => mount.mountPath === '/output'
)
expect(outputMount).toMatchObject({
name: 'recipe-output',
subPath: `workflow-output/test-wf/${runId}`,
})
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-artifact-reader')
expect(createdPodNames).not.toContain('test-wf-mcp-host')
expect(getCreatedArtifactReaderPod(coreApi).spec!.containers![0].volumeMounts).toContainEqual(
expect.objectContaining({
name: 'recipe-output',
mountPath: '/output',
subPath: `workflow-output/test-wf/${runId}`,
})
)
expectWorkflowOutputAnchorAffinity(coordPod)
})
it('rejects invalid snippet step shapes before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'bad', instruction: 'run', run: snippetRun() }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('cannot configure both run and instruction')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects unsupported snippet run fields before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'bad',
run: { ...snippetRun(), unexpected: true } as never,
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('run contains unsupported field "unexpected"')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects non-snippet run types before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{
id: 'bad',
run: { type: 'handler', handler: 'foo' } as never,
},
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('run.type must be snippet')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects cyclic step dependencies before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{ id: 'first', dependsOn: ['second'], run: snippetRun() },
{ id: 'second', dependsOn: ['first'], run: snippetRun() },
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('dependency cycle detected')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects duplicate step ids before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [
{ id: 'first', run: { handler: 'noop' } },
{ id: 'first', run: { handler: 'noop' } },
],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('duplicate step id "first" is not allowed')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('rejects unknown step dependencies before creating pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
agent: undefined,
steps: [{ id: 'first', dependsOn: ['missing'], run: snippetRun() }],
})
)
expect(result.workflowPhase).toBe('failed')
expect(result.message).toContain('depends on unknown step "missing"')
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('does not read selected-node or pin mcp-host pods to stale node names', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
const mcpHostPod = getCreatedMcpHostPod(coreApi)
expectWorkflowOutputAnchorAffinity(mcpHostPod)
})
it('creates mcp-host pods with anchor affinity and does not inspect selected-node', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: { annotations: {} },
spec: { accessModes: ['ReadWriteOnce'] },
status: { accessModes: ['ReadWriteOnce'] },
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).toBeDefined()
expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
const mcpHostPod = getCreatedMcpHostPod(coreApi)
expectWorkflowOutputAnchorAffinity(mcpHostPod)
})
it('does not need selected-node lookup to use anchor affinity', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).toBeDefined()
expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
const mcpHostPod = getCreatedMcpHostPod(coreApi)
expectWorkflowOutputAnchorAffinity(mcpHostPod)
})
it('ignores stale selected-node annotations when creating mcp-host pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
metadata: {
annotations: {
'volume.kubernetes.io/selected-node': 'node-a',
},
},
spec: { accessModes: ['ReadWriteMany'] },
status: { accessModes: ['ReadWriteMany'] },
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).toBeDefined()
expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
const mcpHostPod = getCreatedMcpHostPod(coreApi)
expectWorkflowOutputAnchorAffinity(mcpHostPod)
})
it('creates NetworkPolicies via networkingApi', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(deps.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalled()
})
it('passes runRetention through schedule reconcile into workflow_schedules', async () => {
const { pool: pgPool, query: pgQuery } = makePgPool()
const schedulingDeps = makeDeps({ pgPool })
const reconciler = new WorkflowReconciler(schedulingDeps)
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
triggers: {
schedule: {
cron: '0 9 * * *',
timezone: 'UTC',
},
},
runRetention: {
maxRunDurationSeconds: 1800,
ttlSecondsAfterFinished: 3600,
},
}),
undefined,
undefined,
undefined,
undefined,
'11111111-1111-4111-8111-111111111111'
)
const insertCall = pgQuery.mock.calls.find(([sql]) =>
String(sql).includes('INSERT INTO workflow_schedules')
)
expect(insertCall).toBeDefined()
expect(String(insertCall?.[0])).toContain('max_duration_seconds')
expect(String(insertCall?.[0])).toContain('ttl_seconds_after_finished')
expect((insertCall?.[1] as unknown[] | undefined)?.[2]).toBe(
'11111111-1111-4111-8111-111111111111'
)
expect((insertCall?.[1] as unknown[] | undefined)?.[8]).toBe(1800)
expect((insertCall?.[1] as unknown[] | undefined)?.[9]).toBe(3600)
})
it('mirrors triggers.schedule.concurrencyPolicy into workflow_schedules', async () => {
const { pool: pgPool, query: pgQuery } = makePgPool()
const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({
triggers: {
schedule: { cron: '*/5 * * * *', timezone: 'UTC', concurrencyPolicy: 'Allow' },
},
}),
undefined,
undefined,
undefined,
undefined,
'11111111-1111-4111-8111-111111111111'
)
const insertCall = pgQuery.mock.calls.find(([sql]) =>
String(sql).includes('INSERT INTO workflow_schedules')
)
expect(insertCall).toBeDefined()
expect(String(insertCall?.[0])).toContain('concurrency_policy')
expect((insertCall?.[1] as unknown[] | undefined)?.[10]).toBe('Allow')
})
it('defaults concurrency_policy to Forbid when triggers.schedule omits it', async () => {
const { pool: pgPool, query: pgQuery } = makePgPool()
const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec({ triggers: { schedule: { cron: '*/5 * * * *', suspend: true } } }),
undefined,
undefined,
undefined,
undefined,
'11111111-1111-4111-8111-111111111111'
)
const insertCall = pgQuery.mock.calls.find(([sql]) =>
String(sql).includes('INSERT INTO workflow_schedules')
)
expect((insertCall?.[1] as unknown[] | undefined)?.[10]).toBe('Forbid')
// suspend still maps to enabled = false alongside the policy.
expect((insertCall?.[1] as unknown[] | undefined)?.[6]).toBe(false)
})
it('calls tokenFactory to generate coordinator tokens (WRC→mcp-host tokens are now signed per-request, not on reconcile)', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
const tf = deps.tokenFactory as unknown as ReturnType<typeof makeTokenFactory>
// Coordinator tokens are persistent (Secret-mounted) → signed at reconcile time.
expect(tf.signCoordinatorToMcpHostToken).toHaveBeenCalled()
expect(tf.signCoordinatorToWrcToken).toHaveBeenCalled()
// WRC→mcp-host configure/artifact tokens are no longer issued on reconcile —
// they are signed fresh per call by restEndpoints handlers.
expect(tf.signWrcConfigureToken).not.toHaveBeenCalled()
expect(tf.signWrcArtifactToken).not.toHaveBeenCalled()
})
it('is idempotent — Pod already running causes no crash (readNamespacedPod returns existing)', async () => {
const coreApiWithExistingPod = makeCoreApi(true)
const idempotentDeps = makeDeps({ coreApi: coreApiWithExistingPod as never })
const reconciler = new WorkflowReconciler(idempotentDeps)
// Should not throw on second reconcile
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(result.phase).toBeDefined()
})
it('deletes stale mcp-host and artifact reader before coordinator recovery and recreates runtime pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'test-wf-coordinator') {
return {
status: { phase: 'Failed' },
spec: { containers: [{ image: 'clerum/coordinator:test' }] },
}
}
if (name === 'test-wf-mcp-host') {
return {
status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }] },
spec: { containers: [{ image: 'mcp-host' }] },
}
}
if (name === 'test-wf-workflow-output-anchor') {
return { status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }] } }
}
if (name === 'test-wf-workflow-output-prepare') {
return { status: { phase: 'Succeeded' } }
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
{
workflowExecution: { phase: 'running', attempt: 0 },
},
undefined,
undefined,
runId
)
expect(result.phase).toBeDefined()
const deletedNames = coreApi.deleteNamespacedPod.mock.calls.map(([arg]) => arg.name)
expect(deletedNames.slice(0, 3)).toEqual([
'test-wf-mcp-host',
'test-wf-artifact-reader',
'test-wf-coordinator',
])
const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
([arg]) => arg.body?.metadata?.name
)
expect(createdPodNames).toContain('test-wf-mcp-host')
expect(createdPodNames).toContain('test-wf-artifact-reader')
expect(createdPodNames).toContain('test-wf-coordinator')
})
it('handles spec with no mcpServers (pure-compute workflow)', async () => {
const reconciler = new WorkflowReconciler(deps)
const spec = makeSpec({ mcpServers: undefined })
const result = await reconciler.reconcile('pure-compute', 'uid-pc', 'sandbox-recipes', spec)
expect(result.phase).toBeDefined()
// Should not crash on missing mcpServers
})
it('returns failed when coreApi createNamespacedSecret rejects with non-404', async () => {
const errorDeps = makeDeps({
coreApi: {
...makeCoreApi(),
createNamespacedSecret: vi.fn().mockRejectedValue(new Error('quota exceeded')),
} as never,
})
const reconciler = new WorkflowReconciler(errorDeps)
const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
expect(result.phase).toBe('failed')
expect(result.message).toBeDefined()
})
it('fails reconcile when creating the mcpHost runtime token Secret fails', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-mcp-host-runtime-tokens') {
throw new Error('approval secret create failed')
}
return {}
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).toBe('failed')
expect(result.message).toContain('approval secret create failed')
})
it('refreshes the mcpHost runtime token Secret when stored tokens are near expiry', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-mcp-host-runtime-tokens',
namespace: 'sandbox-recipes',
}),
expect.any(Object)
)
})
it('uses the configured runtime token refresh threshold for mcpHost runtime tokens', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(1200, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(1200, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
makeJwtWithExp(1200, { recipeName: 'test-wf' })
).toString('base64'),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
config: {
...makeConfig(),
runtimeTokenTtlSeconds: 3600,
runtimeTokenRefreshBeforeSeconds: 1800,
} as never,
})
)
await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({ name: 'wf-test-wf-mcp-host-runtime-tokens' }),
expect.any(Object)
)
})
it('reissues mcpHost runtime tokens when the stored recipeName no longer matches the runtime scope', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-child-run-mcp-host-runtime-tokens') {
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'child-run' })
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'child-run' })
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'child-run' })
).toString('base64'),
'mcp-host-gfs-token': makeEncodedGfsAccess(3600, 'child-run'),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'child-run',
makeSpec(),
'parent-recipe'
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'parent-recipe', [])
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-child-run-mcp-host-runtime-tokens',
namespace: 'sandbox-recipes',
body: expect.objectContaining({
data: {
'mcp-host-runtime-access-token': Buffer.from('mcp-host-runtime-access-token').toString(
'base64'
),
'mcp-host-runtime-refresh-token': Buffer.from(
'mcp-host-runtime-refresh-token'
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
'mcp-host-workflow-control-token'
).toString('base64'),
'mcp-host-gfs-token': Buffer.from('gfs-runtime-value').toString('base64'),
},
metadata: expect.objectContaining({
annotations: expect.objectContaining({
'clerum.io/mcp-host-runtime-token-generation': '1',
}),
}),
}),
}),
expect.any(Object)
)
})
it('treats mcpHost runtime token Secret 409 as idempotent and reuses the existing Secret', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
let mcpHostRuntimeSecretCreated = false
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
if (!mcpHostRuntimeSecretCreated) throw { code: 404 }
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
'mcp-host-gfs-token': makeEncodedGfsAccess(),
},
}
}
throw { code: 404 }
})
coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
if (body?.metadata?.name === 'wf-test-wf-mcp-host-runtime-tokens') {
mcpHostRuntimeSecretCreated = true
throw { code: 409 }
}
return {}
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).not.toBe('failed')
expect(coreApi.createNamespacedSecret).toHaveBeenCalled()
expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
})
it('keeps the existing mcpHost runtime token Secret when stored tokens are healthy', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf' })
).toString('base64'),
'mcp-host-gfs-token': makeEncodedGfsAccess(),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const callsBefore = vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'recipe-runtime-ns',
makeSpec(),
undefined,
undefined,
undefined,
runId
)
expect(result.phase).not.toBe('failed')
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length).toBe(callsBefore)
expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
})
it('reissues stale access and refresh scopes once, then converges on the next reconcile', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const expectedWorkflowControlScopes = [
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
]
const canonicalAccessToken = makeJwtWithExp(3600, {
workflowControlScopes: expectedWorkflowControlScopes,
})
const canonicalRefreshToken = makeJwtWithExp(3600, {
workflowControlScopes: expectedWorkflowControlScopes,
})
const canonicalControlToken = makeJwtWithExp(3600, {
scopes: expectedWorkflowControlScopes,
})
vi.mocked(issueMcpHostRuntimeTokens).mockResolvedValue({
accessToken: canonicalAccessToken,
refreshToken: canonicalRefreshToken,
mcpHostControlToken: canonicalControlToken,
})
let secretData: Record<string, string> = {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(3600, { workflowControlScopes: ['workflow:list'] })
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(3600, { workflowControlScopes: ['workflow:list'] })
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
makeJwtWithExp(3600, { scopes: expectedWorkflowControlScopes })
).toString('base64'),
'mcp-host-gfs-token': makeEncodedGfsAccess(),
}
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') return { data: secretData }
throw { code: 404 }
})
coreApi.patchNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
secretData = { ...secretData, ...(body?.data ?? {}) }
return {}
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const spec = makeSpec({
steps: [
{
id: 'broker-trigger',
instruction: 'Trigger a workflow',
allowedTools: { include: ['workflow_trigger'] },
},
],
})
await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', spec)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledTimes(1)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
'recipe-runtime-ns',
'test-wf',
expectedWorkflowControlScopes
)
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)
await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', spec)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledTimes(1)
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)
})
it('reissues only the workflow-control token when declared broker scopes change', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const expectedWorkflowControlScopes = [
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
]
coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
return {
data: {
'mcp-host-runtime-access-token': Buffer.from(
makeJwtWithExp(3600, {
recipeName: 'test-wf',
workflowControlScopes: expectedWorkflowControlScopes,
})
).toString('base64'),
'mcp-host-runtime-refresh-token': Buffer.from(
makeJwtWithExp(3600, {
recipeName: 'test-wf',
workflowControlScopes: expectedWorkflowControlScopes,
})
).toString('base64'),
'mcp-host-workflow-control-token': Buffer.from(
makeJwtWithExp(3600, { recipeName: 'test-wf', scopes: ['workflow:list'] })
).toString('base64'),
'mcp-host-gfs-token': makeEncodedGfsAccess(),
},
}
}
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
const runtimeCallsBefore = vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({
steps: [
{
id: 'broker-trigger',
instruction: 'Trigger a workflow',
allowedTools: { include: ['workflow_trigger'] },
},
],
})
)
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length).toBe(runtimeCallsBefore)
expect(issueMcpHostWorkflowControlToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
...expectedWorkflowControlScopes,
])
expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
name: 'wf-test-wf-mcp-host-runtime-tokens',
namespace: 'sandbox-recipes',
body: expect.objectContaining({
data: {
'mcp-host-workflow-control-token': Buffer.from(
'mcp-host-workflow-control-token'
).toString('base64'),
},
metadata: expect.objectContaining({
annotations: expect.objectContaining({
'clerum.io/mcp-host-runtime-token-generation': '1',
}),
}),
}),
}),
expect.any(Object)
)
})
it('can repair a missing mcpHost runtime token Secret without recreating workflow pods', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', makeSpec())
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
expect(coreApi.createNamespacedSecret).toHaveBeenCalledWith(
expect.objectContaining({
namespace: 'sandbox-recipes',
body: expect.objectContaining({
metadata: expect.objectContaining({
name: 'wf-test-wf-mcp-host-runtime-tokens',
}),
}),
})
)
expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('derives workflow-control scopes from declared workflow broker tools', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({
steps: [
{
id: 'broker',
instruction: 'Use workflow broker',
allowedTools: {
include: [
'clerum__list_workflows',
'clerum__read_workflow',
'clerum__trigger_workflow',
],
},
},
],
})
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
'workflow:list',
'workflow:read',
'workflow:trigger',
'workflow:approval:resolve',
'workflow:approval:decide',
])
})
it('derives write-scoped GFS host tokens from declared publish targets', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({
gfs: {
publishTargets: [{ drive: 'main', target: 'outputs' }],
},
})
)
expect(mintRecipeHostGfsToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', {
scopes: ['gfs.read', 'gfs.write'],
})
})
it('derives workflow approval scopes from agentic approval-gated steps', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({
agent: { provider: 'zai', model: 'glm-4.7' },
steps: [
{
id: 'approval-gated',
instruction: 'Continue after approval',
requiresApproval: {
target: { userId: 'user-1' },
message: 'Approve this step',
},
},
],
})
)
expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
'workflow:approval:resolve',
'workflow:approval:decide',
])
})
it('does not create mcpHost runtime credentials for snippet-only workflows', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.ensureMcpHostRuntimeCredentials(
'recipe-runtime-ns',
'test-wf',
makeSpec({ steps: [{ id: 's1', run: snippetRun() }] })
)
expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
})
it('cleans workflow output through artifact-reader before deleting runtime resources', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedService = vi.fn().mockImplementation(({ name }: { name: string }) => {
if (name === 'wf-test-wf-artifact-reader') return Promise.resolve({})
throw { code: 404 }
})
const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 204 } as Response)
vi.stubGlobal('fetch', fetchMock)
const tokenFactory = makeTokenFactory()
const reconciler = new WorkflowReconciler(
makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
)
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')
expect(tokenFactory.signWrcArtifactDeleteToken).toHaveBeenCalledWith(
'test-wf',
'sandbox-recipes'
)
expect(fetchMock).toHaveBeenCalledWith(
'http://wf-test-wf-artifact-reader.sandbox-recipes.svc.cluster.local:8080/api/v1/workflow/artifacts',
expect.objectContaining({
method: 'DELETE',
headers: { authorization: 'Bearer wrc-artifact-delete-token' },
})
)
expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith(
expect.objectContaining({ name: 'test-wf-artifact-reader' })
)
})
it('keeps the finalizer retryable when runtime pod cleanup fails', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.deleteNamespacedPod = vi.fn().mockImplementation(({ name }: { name: string }) => {
if (name === 'test-wf-workflow-output-anchor') {
return Promise.reject({ code: 500, message: 'api server unavailable' })
}
return Promise.resolve({})
})
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await expect(reconciler.reconcileDelete('test-wf', 'sandbox-recipes')).rejects.toThrow(
'test-wf-workflow-output-anchor'
)
expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith(
expect.objectContaining({ name: 'test-wf-coordinator' })
)
expect(coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
name: 'test-wf-workflow-output',
namespace: 'sandbox-recipes',
})
})
it('falls back to mcp-host artifact cleanup when artifact-reader is unavailable', async () => {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedService = vi.fn().mockImplementation(({ name }: { name: string }) => {
if (name === 'wf-test-wf-mcp-host') return Promise.resolve({})
throw { code: 404 }
})
const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 204 } as Response)
vi.stubGlobal('fetch', fetchMock)
const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')
expect(fetchMock).toHaveBeenCalledTimes(1)
expect(String(fetchMock.mock.calls[0][0])).toBe(
'http://wf-test-wf-mcp-host.sandbox-recipes.svc.cluster.local:8080/api/v1/workflow/artifacts'
)
})
it('deletes workflow trigger policy and cancels live approvals for the deleted recipe', async () => {
const { pool: pgPool, query, release } = makePgPool()
query.mockImplementation(async (sql: string) => {
if (String(sql).includes("to_regclass('public.workflow_recipe_allowed_users')")) {
return { rows: [{ regclass: 'workflow_recipe_allowed_users' }], rowCount: 1 }
}
const trimmed = String(sql).trim().toUpperCase()
if (trimmed.startsWith('SELECT')) {
return { rows: [], rowCount: 0 }
}
return { rows: [], rowCount: 1 }
})
const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')
const sqls = query.mock.calls.map(([sql]) => String(sql))
expect(sqls).toContainEqual('BEGIN')
expect(sqls).toContainEqual(expect.stringContaining('DELETE FROM user_workflow_triggers'))
expect(sqls).toContainEqual(expect.stringContaining('DELETE FROM team_workflow_triggers'))
expect(sqls).toContainEqual(
expect.stringContaining('DELETE FROM workflow_recipe_allowed_teams')
)
expect(sqls).toContainEqual(
expect.stringContaining("to_regclass('public.workflow_recipe_allowed_users')")
)
expect(sqls).toContainEqual(
expect.stringContaining('DELETE FROM workflow_recipe_allowed_users')
)
expect(sqls).toContainEqual(expect.stringContaining('UPDATE workflow_approval_requests'))
expect(sqls).toContainEqual(expect.stringContaining("status IN ('pending', 'approved')"))
expect(sqls).toContainEqual(
expect.stringContaining("cancelled_by = COALESCE(cancelled_by, 'workflow-recipe-delete')")
)
expect(sqls).toContainEqual('COMMIT')
expect(query.mock.calls).toContainEqual([
expect.stringContaining('DELETE FROM team_workflow_triggers'),
['sandbox-recipes', 'test-wf'],
])
expect(query.mock.calls).toContainEqual([
expect.stringContaining('DELETE FROM workflow_recipe_allowed_teams'),
['sandbox-recipes', 'test-wf'],
])
expect(query.mock.calls).toContainEqual([
expect.stringContaining('DELETE FROM workflow_recipe_allowed_users'),
['sandbox-recipes', 'test-wf'],
])
expect(query.mock.calls).toContainEqual([
expect.stringContaining('UPDATE workflow_approval_requests'),
['sandbox-recipes', 'test-wf'],
])
expect(release).toHaveBeenCalledTimes(1)
})
it('keeps recipe-delete cleanup idempotent when legacy allowed-users table is absent', async () => {
const { pool: pgPool, query, release } = makePgPool()
const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))
await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')
const sqls = query.mock.calls.map(([sql]) => String(sql))
expect(sqls).toContainEqual(
expect.stringContaining("to_regclass('public.workflow_recipe_allowed_users')")
)
expect(sqls).not.toContainEqual(
expect.stringContaining('DELETE FROM workflow_recipe_allowed_users')
)
expect(sqls).toContainEqual('COMMIT')
expect(release).toHaveBeenCalledTimes(1)
})
it('workflowPhase in result is a valid WorkflowPhase value', async () => {
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'test-wf',
'uid-123',
'sandbox-recipes',
makeSpec(),
undefined,
undefined,
'test-wf',
runId
)
const validPhases = [
'pending',
'approved',
'deploying',
'running',
'awaiting_approval',
'completed',
'failed',
'cancelling',
'cancelled',
'recovering',
'initializing',
]
expect(validPhases).toContain(result.workflowPhase)
})
})
describe('cleanupPluginWorkloadSdk legacy internet policy ownership', () => {
const RECIPE = 'sdk-only'
const RECIPE_UID = 'uid-sdk-only'
const LEGACY = `${RECIPE}-mcp-servers-egress-internet`
function legacyDeletes(networkingApi: ReturnType<typeof makeNetworkingApi>): number {
return networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === LEGACY
).length
}
function makeCleanupCoreApi(): ReturnType<typeof makeCoreApi> {
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
return coreApi
}
it('routes a labeled legacy policy through the shared live ledger instead of the label sweep', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
let legacyPresent = true
networkingApi.listNamespacedNetworkPolicy.mockImplementation(
async ({ namespace }: { namespace: string }) => ({
items:
namespace === 'mcp-server'
? legacyPresent
? [
{
metadata: {
name: LEGACY,
namespace,
labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
},
},
]
: []
: [],
})
)
networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
if (name === LEGACY) legacyPresent = false
return {}
})
const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: coreApi as never,
networkingApi: networkingApi as never,
pluginWorkloadSdkRevocationClient: {
revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
} as never,
})
)
await reconciler.cleanupPluginWorkloadSdk(RECIPE, { recipeUid: RECIPE_UID })
// Liveness witness: both label-sweep namespaces were evaluated, and the
// shared by-name path sent the sole legacy DELETE.
expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
expect.objectContaining({
namespace: 'mcp-server',
labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,
})
)
expect(legacyDeletes(networkingApi)).toBe(1)
await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
expect(legacyDeletes(networkingApi)).toBe(1)
})
it('keeps live cleanup on the shared backoff while finalizer cleanup gets one last chance', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
try {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
let legacyDeleteFails = true
networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
if (name === LEGACY && legacyDeleteFails) throw { code: 403, message: 'forbidden' }
return {}
})
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: makeCleanupCoreApi() as never,
networkingApi: networkingApi as never,
pluginWorkloadSdkRevocationClient: {
revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
} as never,
})
)
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
).resolves.toBe('pending')
expect(legacyDeletes(networkingApi)).toBe(1)
const pending = await reconciler
.cleanupPluginWorkloadSdk(RECIPE, {
recipeUid: RECIPE_UID,
})
.catch(error => error)
expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(
LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS
)
expect(legacyDeletes(networkingApi)).toBe(1)
legacyDeleteFails = false
await reconciler.cleanupPluginWorkloadSdk(RECIPE, {
recipeUid: RECIPE_UID,
recipeDeleted: true,
})
expect(legacyDeletes(networkingApi)).toBe(2)
await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
expect(legacyDeletes(networkingApi)).toBe(3)
} finally {
vi.useRealTimers()
}
})
it('keeps the legacy policy live when cleanup preserves the workflow runtime', async () => {
const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
const reconciler = new WorkflowReconciler(
makeDeps({
coreApi: makeCleanupCoreApi() as never,
networkingApi: networkingApi as never,
pluginWorkloadSdkRevocationClient: {
revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
} as never,
})
)
await reconciler.cleanupPluginWorkloadSdk('hybrid', {
recipeUid: 'uid-hybrid',
preserveWorkflowRuntime: true,
})
expect(
networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'hybrid-mcp-servers-egress-internet'
)
).toHaveLength(0)
expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
})
describe('buildNetworkPolicyOwnershipConditions', () => {
it('names the conflicting policies in sorted order, whatever order the apply produced them in', () => {
const conditions = buildNetworkPolicyOwnershipConditions(
{
conflicts: [
{ policy: 'test-wf-wrc-to-artifact-reader', reason: 'identity-label-mismatch' },
{ policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' },
],
retryPending: false,
},
'2026-09-23T10:00:00.000Z'
)
expect(conditions).toEqual([
{
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message:
'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch), test-wf-wrc-to-artifact-reader (identity-label-mismatch)',
lastTransitionTime: '2026-09-23T10:00:00.000Z',
},
])
})
})
// The one translation of a full pass summary into status fields, shared by
// the run lane, the eager lane and the SDK-only lane (the `reconcile` writer
// of Table A). `toStrictEqual` pins which keys are present: an absent
// ownership key keeps the published condition, so it must not be emitted as
// an explicit `undefined`.
describe('translateNetworkPolicyPassSummary', () => {
const now = '2026-09-23T12:00:00.000Z'
const earlier = '2026-09-20T08:00:00.000Z'
const pass = (overrides: Partial<NetworkPolicyPassSummary> = {}): NetworkPolicyPassSummary => ({
conflicts: [],
retryPending: false,
prune: 'converged',
legacy: 'removed',
...overrides,
})
const conflictSummary = pass({
conflicts: [{ policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' as const }],
})
const publishedConflict = {
type: 'WorkflowNetworkPolicyOwnership',
status: 'False' as const,
reason: 'OwnershipConflict',
message: 'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
lastTransitionTime: earlier,
}
const publishedRetryMarker = {
type: 'WorkflowNetworkPoliciesConverged',
status: 'False' as const,
reason: 'RetryPending',
message: 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
lastTransitionTime: earlier,
}
it('emits neither field when the pass produced no summary, so the published condition is kept', () => {
const result = translateNetworkPolicyPassSummary(undefined, [publishedConflict], now)
expect(result).toStrictEqual({})
// Witness: the same existing conditions with a summary do produce the
// field, so the empty result above is caused by the missing summary.
expect(
translateNetworkPolicyPassSummary(conflictSummary, [publishedConflict], now)
.networkPolicyOwnershipConditions
).toEqual([publishedConflict])
})
it('clears the condition with [] and sets no retry flag when nothing conflicts or is pending', () => {
const result = translateNetworkPolicyPassSummary(pass(), [publishedConflict], now)
expect(result.networkPolicyOwnershipConditions).toEqual([])
expect(result).not.toHaveProperty('networkPolicyRetryPending')
expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
})
it('replaces the conflict with the retry marker and sets the retry flag when a retry is pending without a conflict', () => {
const result = translateNetworkPolicyPassSummary(
pass({ retryPending: true }),
[publishedConflict],
now
)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [{ ...publishedRetryMarker, lastTransitionTime: now }],
networkPolicyRetryPending: true,
})
})
it('keeps the lastTransitionTime of a published retry marker while the retry stays pending', () => {
const result = translateNetworkPolicyPassSummary(
pass({ retryPending: true }),
[publishedRetryMarker],
now
)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [publishedRetryMarker],
networkPolicyRetryPending: true,
})
})
it('clears a published retry marker with [] once no retry is pending', () => {
const result = translateNetworkPolicyPassSummary(pass(), [publishedRetryMarker], now)
expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
})
it('replaces the condition with one False condition naming every conflicting policy', () => {
const result = translateNetworkPolicyPassSummary(
pass({
conflicts: [
{ policy: 'test-wf-wrc-to-artifact-reader', reason: 'identity-label-mismatch' },
{ policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' },
],
}),
undefined,
now
)
expect(result.networkPolicyOwnershipConditions).toEqual([
{
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message:
'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch), test-wf-wrc-to-artifact-reader (identity-label-mismatch)',
lastTransitionTime: now,
},
])
expect(result).not.toHaveProperty('networkPolicyRetryPending')
})
it('keeps the lastTransitionTime of an identical published condition', () => {
const result = translateNetworkPolicyPassSummary(conflictSummary, [publishedConflict], now)
expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [publishedConflict] })
expect(result.networkPolicyOwnershipConditions?.[0]?.lastTransitionTime).toBe(earlier)
})
it('reports a conflict and a pending retry from the same pass together', () => {
const result = translateNetworkPolicyPassSummary(
{ ...conflictSummary, retryPending: true },
undefined,
now
)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [
{ ...publishedConflict, lastTransitionTime: now },
{ ...publishedRetryMarker, lastTransitionTime: now },
],
networkPolicyRetryPending: true,
})
})
// R3-L3: a prune that could not delete (a non-404 DELETE) is its own fact;
// a failed LIST is `unevaluated` (R4-L6). It is published as PrunePending
// and never sets the retry flag: nothing in the run's short-circuits can
// prune, so a requeue would only repeat the failed DELETE.
describe('prune-pending', () => {
const publishedPruneMarker = {
type: 'WorkflowNetworkPoliciesConverged',
status: 'False' as const,
reason: 'PrunePending',
message:
'One or more run-lane NetworkPolicies the spec no longer wants are pending a delete; the next reconcile() pass or the finalizer removes them',
lastTransitionTime: earlier,
}
it('publishes PrunePending without the retry flag when only the prune is pending', () => {
const result = translateNetworkPolicyPassSummary(
pass({ prune: 'pending' }),
[publishedConflict],
now
)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [{ ...publishedPruneMarker, lastTransitionTime: now }],
})
})
it('clears a published PrunePending marker once a pass pruned', () => {
const result = translateNetworkPolicyPassSummary(pass(), [publishedPruneMarker], now)
expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
})
it('reports a pending retry and a pending prune in one RetryAndPrunePending marker', () => {
const result = translateNetworkPolicyPassSummary(
pass({ retryPending: true, prune: 'pending' }),
[publishedPruneMarker],
now
)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [
{
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'RetryAndPrunePending',
message:
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
lastTransitionTime: earlier,
},
],
networkPolicyRetryPending: true,
})
})
})
// R4-L6: a failed prune LIST evaluated nothing, so the prune fact is
// `unevaluated`, never `pending`. It has its own reasons and sets no retry
// flag of its own.
describe('prune-unevaluated', () => {
const convergedMarkers = (result: ReturnType<typeof translateNetworkPolicyPassSummary>) =>
(result.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowNetworkPoliciesConverged'
)
it('R4-L6: publishes PruneUnevaluated without the retry flag when the prune LIST failed', () => {
const result = translateNetworkPolicyPassSummary(
pass({ prune: 'unevaluated' }),
[publishedConflict],
now
)
expect(convergedMarkers(result)).toMatchObject([
{ status: 'False', reason: 'PruneUnevaluated', lastTransitionTime: now },
])
expect(result.networkPolicyRetryPending).toBeUndefined()
})
it('R4-L6: publishes RetryAndPruneUnevaluated for a pending retry and an unevaluated prune', () => {
const result = translateNetworkPolicyPassSummary(
pass({ retryPending: true, prune: 'unevaluated' }),
[],
now
)
expect(convergedMarkers(result)).toMatchObject([
{ status: 'False', reason: 'RetryAndPruneUnevaluated' },
])
expect(result.networkPolicyRetryPending).toBe(true)
})
})
// R4-L3: the legacy policy is its own fact, on its own condition.
describe('legacy', () => {
const legacyMarkers = (result: ReturnType<typeof translateNetworkPolicyPassSummary>) =>
(result.networkPolicyOwnershipConditions ?? []).filter(
c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
)
it('R4-L3: publishes DeletePending on its own condition, apart from the prune marker', () => {
const result = translateNetworkPolicyPassSummary(pass({ legacy: 'pending' }), [], now)
expect(result).toStrictEqual({
networkPolicyOwnershipConditions: [
{
type: 'WorkflowLegacyNetworkPolicyRemoved',
status: 'False',
reason: 'DeletePending',
message:
'The legacy mcp-servers internet egress NetworkPolicy is pending a delete; a later pass or the finalizer removes it',
lastTransitionTime: now,
},
],
})
})
it('R4-L3: keeps the first transition time while the delete stays pending, and clears it once removed', () => {
const published = translateNetworkPolicyPassSummary(
pass({ legacy: 'pending' }),
[],
earlier
).networkPolicyOwnershipConditions!
// Liveness witness: the failing pass published the marker.
expect(legacyMarkers({ networkPolicyOwnershipConditions: published })).toHaveLength(1)
const still = translateNetworkPolicyPassSummary(pass({ legacy: 'pending' }), published, now)
expect(legacyMarkers(still)).toMatchObject([{ lastTransitionTime: earlier }])
expect(translateNetworkPolicyPassSummary(pass(), published, now)).toStrictEqual({
networkPolicyOwnershipConditions: [],
})
})
})
})
// R4-L3: every writer goes through `networkPolicyMarkerConditions`, and each
// fact is observed, carried or cleared as Table A says.
describe('networkPolicyMarkerConditions', () => {
const now = '2026-09-23T12:00:00.000Z'
const earlier = '2026-09-20T08:00:00.000Z'
const pass = (overrides: Partial<NetworkPolicyPassSummary> = {}): NetworkPolicyPassSummary => ({
conflicts: [],
retryPending: false,
prune: 'converged',
legacy: 'removed',
...overrides,
})
const conflict = { policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' as const }
const published = (overrides: Partial<NetworkPolicyPassSummary>) =>
networkPolicyMarkerConditions({ kind: 'reconcile', summary: pass(overrides) }, [], earlier)
const reasons = (conditions: { type: string; reason?: string }[]) =>
conditions.map(c => `${c.type}/${c.reason}`)
describe('apply-retry', () => {
it('R4-L3: observes the apply and carries the prune and legacy facts', () => {
const existing = published({ retryPending: true, prune: 'pending', legacy: 'pending' })
// Liveness witness: all three facts were published.
expect(reasons(existing)).toEqual([
'WorkflowNetworkPoliciesConverged/RetryAndPrunePending',
'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
])
const settled = networkPolicyMarkerConditions(
{ kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },
existing,
now
)
expect(reasons(settled)).toEqual([
'WorkflowNetworkPoliciesConverged/PrunePending',
'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
])
expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)
})
it('R4-L3: carries an unevaluated prune through a pending retry', () => {
const existing = published({ prune: 'unevaluated' })
const retried = networkPolicyMarkerConditions(
{ kind: 'apply-retry', summary: { conflicts: [], retryPending: true } },
existing,
now
)
expect(reasons(retried)).toEqual([
'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',
])
})
it('R4-L3: publishes the conflicts the retry observed', () => {
const retried = networkPolicyMarkerConditions(
{ kind: 'apply-retry', summary: { conflicts: [conflict], retryPending: false } },
[],
now
)
expect(reasons(retried)).toEqual(['WorkflowNetworkPolicyOwnership/OwnershipConflict'])
})
})
describe('legacy-retry', () => {
it('R4-L4: clears the legacy fact once removed and carries the apply, the prune and the ownership condition', () => {
const existing = published({
conflicts: [conflict],
retryPending: true,
prune: 'unevaluated',
legacy: 'pending',
})
// Liveness witness: all three conditions were published.
expect(existing).toHaveLength(3)
const settled = networkPolicyMarkerConditions(
{ kind: 'legacy-retry', legacy: 'removed' },
existing,
now
)
expect(reasons(settled)).toEqual([
'WorkflowNetworkPolicyOwnership/OwnershipConflict',
'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',
])
expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)
})
it('R4-L4: keeps DeletePending and its transition time while the delete stays pending', () => {
const existing = published({ prune: 'pending', legacy: 'pending' })
expect(existing).toHaveLength(2)
expect(
networkPolicyMarkerConditions({ kind: 'legacy-retry', legacy: 'pending' }, existing, now)
).toEqual(existing)
})
it('R4-L4: enters only for a published DeletePending', () => {
expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ legacy: 'pending' }))).toBe(true)
expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ prune: 'pending' }))).toBe(false)
expect(hasLegacyNetworkPolicyDeletePendingMarker(undefined)).toBe(false)
expect(
hasLegacyNetworkPolicyDeletePendingMarker([
{
type: 'WorkflowLegacyNetworkPolicyRemoved',
status: 'False',
reason: 'SomethingElse',
message: 'written by another head',
lastTransitionTime: earlier,
},
])
).toBe(false)
})
})
describe('terminal', () => {
it('R4-L3: drops a pending apply and carries the prune, the legacy fact and the ownership condition', () => {
const existing = published({
conflicts: [conflict],
retryPending: true,
prune: 'pending',
legacy: 'pending',
})
// Liveness witness: all three conditions were published.
expect(existing).toHaveLength(3)
const settled = networkPolicyMarkerConditions({ kind: 'terminal' }, existing, now)
expect(reasons(settled)).toEqual([
'WorkflowNetworkPolicyOwnership/OwnershipConflict',
'WorkflowNetworkPoliciesConverged/PrunePending',
'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
])
})
it('R4-L3: removes a marker that only named a pending apply', () => {
const existing = published({ retryPending: true })
expect(reasons(existing)).toEqual(['WorkflowNetworkPoliciesConverged/RetryPending'])
expect(networkPolicyMarkerConditions({ kind: 'terminal' }, existing, now)).toEqual([])
})
})
describe('unmanaged', () => {
it('R4-L3: clears every fact after a full SDK teardown', () => {
const existing = published({
conflicts: [conflict],
retryPending: true,
prune: 'unevaluated',
legacy: 'pending',
})
// Liveness witness: the group was published.
expect(existing).toHaveLength(3)
expect(networkPolicyMarkerConditions({ kind: 'unmanaged' }, existing, now)).toEqual([])
})
})
describe('a reason this controller did not write', () => {
const foreignMarker = {
type: 'WorkflowNetworkPoliciesConverged',
status: 'False' as const,
reason: 'SomethingElse',
message: 'written by another head',
lastTransitionTime: earlier,
}
const foreignLegacy = {
type: 'WorkflowLegacyNetworkPolicyRemoved',
status: 'False' as const,
reason: 'SomethingElse',
message: 'written by another head',
lastTransitionTime: earlier,
}
it('R4-L3: is kept as it is by a carrying writer, which warns', () => {
const warnLog = captureLogger('warn')
try {
const carried = networkPolicyMarkerConditions(
{ kind: 'terminal' },
[foreignMarker, foreignLegacy],
now
)
expect(carried).toEqual([foreignMarker, foreignLegacy])
expect(warnLog).toHaveBeenCalledTimes(2)
expect(warnLog).toHaveBeenCalledWith(
'Keeping a NetworkPolicy marker with a reason this controller did not write',
{
writer: 'terminal',
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'SomethingElse',
}
)
} finally {
warnLog.mockRestore()
}
})
it('R4-L3: is overwritten by reconcile()', () => {
const warnLog = captureLogger('warn')
try {
const overwritten = networkPolicyMarkerConditions(
{ kind: 'reconcile', summary: pass({ prune: 'pending' }) },
[foreignMarker, foreignLegacy],
now
)
expect(reasons(overwritten)).toEqual(['WorkflowNetworkPoliciesConverged/PrunePending'])
expect(warnLog).not.toHaveBeenCalled()
// Liveness witness: the same input through a carrying writer warns.
networkPolicyMarkerConditions({ kind: 'terminal' }, [foreignMarker], now)
expect(warnLog).toHaveBeenCalledTimes(1)
} finally {
warnLog.mockRestore()
}
})
})
})
// R4-L5: each combination of facts has its own reason, and the facts are read
// back from the reason alone. A reworded message must not change what a
// published marker means. R4-L6: the prune fact has three values (Table A),
// so there are five combinations with a marker.
describe('networkPolicyMarkerFacts', () => {
const now = '2026-09-23T12:00:00.000Z'
const combinations = [
{ apply: 'pending', prune: 'converged' },
{ apply: 'converged', prune: 'pending' },
{ apply: 'converged', prune: 'unevaluated' },
{ apply: 'pending', prune: 'pending' },
{ apply: 'pending', prune: 'unevaluated' },
] as const
it('R4-L5: a pending apply and a pending prune publish their own reason', () => {
const marker = buildNetworkPolicyConvergedCondition({ apply: 'pending', prune: 'pending' }, now)
expect(marker?.reason).toBe('RetryAndPrunePending')
})
it('R4-L6: an unevaluated prune publishes its own reason, alone or with a pending apply', () => {
expect(
buildNetworkPolicyConvergedCondition({ apply: 'converged', prune: 'unevaluated' }, now)
?.reason
).toBe('PruneUnevaluated')
expect(
buildNetworkPolicyConvergedCondition({ apply: 'pending', prune: 'unevaluated' }, now)?.reason
).toBe('RetryAndPruneUnevaluated')
})
it('R4-L6: converged facts publish no marker and read back from no marker', () => {
expect(
buildNetworkPolicyConvergedCondition({ apply: 'converged', prune: 'converged' }, now)
).toBeUndefined()
expect(networkPolicyMarkerFacts([])).toStrictEqual({ apply: 'converged', prune: 'converged' })
})
it.each(combinations)(
'R4-L5: reads %o back from the reason alone, whatever the message says',
facts => {
const marker = buildNetworkPolicyConvergedCondition(facts, now)
// Liveness witness: the builder published a marker for these facts.
expect(marker?.type).toBe('WorkflowNetworkPoliciesConverged')
expect(marker?.status).toBe('False')
expect(networkPolicyMarkerFacts([marker!])).toStrictEqual(facts)
expect(networkPolicyMarkerFacts([{ ...marker!, message: 'reworded' }])).toStrictEqual(facts)
}
)
it('R4-L5: every combination has a distinct reason', () => {
const reasons = combinations.map(
facts => buildNetworkPolicyConvergedCondition(facts, now)?.reason
)
expect(new Set(reasons).size).toBe(combinations.length)
})
it('reads the one historical RetryPending message that encoded a pending prune', () => {
const historical = {
type: 'WorkflowNetworkPoliciesConverged',
status: 'False' as const,
reason: 'RetryPending',
message:
'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
lastTransitionTime: now,
}
expect(networkPolicyMarkerFacts([historical])).toStrictEqual({
apply: 'pending',
prune: 'pending',
})
expect(
networkPolicyMarkerFacts([{ ...historical, message: `${historical.message} ` }])
).toStrictEqual({
apply: 'pending',
prune: 'converged',
})
const settled = networkPolicyMarkerConditions(
{ kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },
[historical],
now
)
expect(settled).toMatchObject([
{
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'PrunePending',
},
])
})
})
describe('networkPolicyConditionsChanged', () => {
const conflict = {
type: 'WorkflowNetworkPolicyOwnership',
status: 'False' as const,
reason: 'OwnershipConflict',
message: 'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
lastTransitionTime: '2026-09-20T08:00:00.000Z',
}
const marker = {
type: 'WorkflowNetworkPoliciesConverged',
status: 'False' as const,
reason: 'RetryPending',
message: 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
lastTransitionTime: '2026-09-20T08:00:00.000Z',
}
const unrelated = {
type: 'Ready',
status: 'True' as const,
reason: 'Ready',
message: 'ready',
lastTransitionTime: '2026-09-20T08:00:00.000Z',
}
it('reports no change when the group matches, whatever lastTransitionTime or other types say', () => {
expect(
networkPolicyConditionsChanged(
[conflict, marker, unrelated],
[
{ ...conflict, lastTransitionTime: '2026-09-24T00:00:00.000Z' },
{ ...marker, lastTransitionTime: '2026-09-24T00:00:00.000Z' },
]
)
).toBe(false)
})
it('reports a change when the retry marker appears', () => {
expect(networkPolicyConditionsChanged([conflict], [conflict, marker])).toBe(true)
})
it('reports a change when the retry marker goes away', () => {
expect(networkPolicyConditionsChanged([marker, unrelated], [])).toBe(true)
})
it('reports a change when the ownership condition message changes', () => {
expect(networkPolicyConditionsChanged([conflict], [{ ...conflict, message: 'other' }])).toBe(
true
)
})
})