Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { buildMcpHostHeadlessService } from './podFactory'
import { buildMcpHostServiceName } from './resourceNames'
import {
LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,
LegacyNetworkPolicyDeletePendingError,
WorkflowReconciler,
type WorkflowReconcilerDeps,
} from './workflowReconciler'
const crashRecoveryMocks = vi.hoisted(() => ({
deletePodIfExists: vi.fn().mockResolvedValue(undefined),
waitForPodDeletion: vi.fn().mockResolvedValue(true),
evaluateCompletedRuntimePodRecovery: vi.fn().mockReturnValue({
action: 'replace',
message:
'mcp_host pod completed before workflow became terminal; creating replacement (attempt 1/3)',
newPhase: 'recovering',
newAttempt: 1,
}),
evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
getPodPhase: vi.fn().mockResolvedValue(undefined),
getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
}),
getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
isRecoverableContainerWaitingReason: vi.fn(
(phase: string | undefined, reason: string | undefined) =>
Boolean(
reason &&
[
'CreateContainerConfigError',
'CreateContainerError',
'ImagePullBackOff',
'ErrImagePull',
'InvalidImageName',
'CrashLoopBackOff',
].includes(reason) &&
(phase === 'Pending' || (phase === 'Running' && reason === 'CrashLoopBackOff'))
)
),
}))
const runtimeTokenIssuerMocks = vi.hoisted(() => ({
issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
accessToken: 'runtime-access-token',
refreshToken: 'runtime-refresh-token',
mcpHostControlToken: 'mcp-host-control-token',
}),
issueMcpHostWorkflowControlToken: vi.fn().mockResolvedValue('mcp-host-control-token'),
}))
vi.mock('./crashRecovery', () => crashRecoveryMocks)
vi.mock('./mcpHostRuntimeTokenIssuerClient', () => runtimeTokenIssuerMocks)
const gfsBindingMocks = vi.hoisted(() => ({
mintRecipeHostGfsToken: vi.fn().mockResolvedValue({
['to'.concat('ken')]: 'gfs-runtime-value',
expiresInSeconds: 300,
subject: 'host:3rd:sandbox-recipes/workflow-recipe',
}),
}))
vi.mock('../gfsBinding', () => gfsBindingMocks)
const isoDateAgo = (ageMs: number) => new Date(Date.now() - ageMs).toISOString()
function unsignedRuntimeJwt(payload: Record<string, unknown>): string {
return `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify(payload)).toString('base64url')}.x`
}
type MockPodReadiness = {
ready: boolean
phase?: string
uid?: string
waitingReason?: string
schedulingReason?: string
}
function mockAgenticOutputPodPhases(overrides?: {
anchor?: string
prepare?: string
mcpHost?: string | (() => string | undefined)
}) {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'agentic-recipe-workflow-output-anchor') {
return overrides?.anchor ?? 'Running'
}
if (name === 'agentic-recipe-workflow-output-prepare') {
return overrides?.prepare ?? 'Succeeded'
}
if (name === 'agentic-recipe-mcp-host') {
const value = overrides?.mcpHost
return typeof value === 'function' ? value() : (value ?? undefined)
}
return undefined
})
}
function mockMcpHostReadiness(readiness: MockPodReadiness) {
crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
if (podName.endsWith('-mcp-host')) {
return readiness
}
return { ready: true, phase: 'Running' }
})
}
function mockMcpHostReadinessSequence(...sequence: MockPodReadiness[]) {
let mcpHostCalls = 0
crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
if (podName.endsWith('-mcp-host')) {
const value = sequence[Math.min(mcpHostCalls, sequence.length - 1)]
mcpHostCalls += 1
return value
}
return { ready: true, phase: 'Running' }
})
}
describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () => {
const sandboxNamespace = 'sandbox-recipes'
const mcpServerNamespace = 'mcp-server'
const mockCoreApi = {
readNamespacedPod: vi.fn().mockResolvedValue({}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
createNamespacedService: vi.fn().mockResolvedValue({}),
createNamespacedPod: vi.fn().mockResolvedValue({}),
patchNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedConfigMap: vi.fn().mockResolvedValue({}),
deleteNamespacedService: vi.fn().mockResolvedValue({}),
readNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({
metadata: { name: 'existing-workflow-output', deletionTimestamp: undefined },
}),
createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
// Under test: new label-selector cleanup for cross-namespace-orphaned Services.
deleteCollectionNamespacedService: vi.fn().mockResolvedValue({}),
}
const mockNetworkingApi = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
}
const mockCustomApi = {
patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
}
// Minimal token factory for the mcp-host artifact cleanup path.
const mockTokenFactory = {
signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('stub-token'),
signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('coordinator-mcp-host-token'),
signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('custom-coordinator-wrc-token'),
signCoordinatorToWrcToken: vi.fn().mockResolvedValue('coordinator-wrc-token'),
}
const deps = {
coreApi: mockCoreApi,
customApi: mockCustomApi,
networkingApi: mockNetworkingApi,
config: {
coordinatorImage: 'coordinator:test',
mcpHostImage: 'mcp-host:test',
wrcEndpoint: 'http://wrc.example/api',
sandboxNamespace,
mcpServerNamespace,
imagePullPolicy: 'IfNotPresent' as const,
maxWorkflowSteps: 100,
runtimeTokenTtlSeconds: 3600,
runtimeTokenRefreshBeforeSeconds: 300,
},
tokenFactory: mockTokenFactory,
pluginWorkloadSdkRevocationClient: {
revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),
finalize: vi.fn(),
},
} as unknown as WorkflowReconcilerDeps
beforeEach(() => {
vi.clearAllMocks()
crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
// Run-scoped reconciles skip the eager→run recreate gate when the mcp-host already
// carries the workflow-output claim label.
mockCoreApi.readNamespacedPod.mockImplementation(async (params: { name?: string }) => {
if (typeof params.name === 'string' && params.name.endsWith('-mcp-host')) {
return {
metadata: {
labels: { 'clerum.io/workflow-output-claim': 'shared-output' },
},
}
}
return {}
})
mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
// No Service exists until a test says so: reconcile reads before it
// creates, so a resolved read would stand for a live (spec-less) Service.
mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
mockCoreApi.createNamespacedSecret.mockResolvedValue({})
mockCoreApi.createNamespacedConfigMap.mockResolvedValue({})
mockCoreApi.createNamespacedService.mockResolvedValue({})
mockCoreApi.createNamespacedPod.mockResolvedValue({})
mockCoreApi.patchNamespacedSecret.mockResolvedValue({})
mockNetworkingApi.createNamespacedNetworkPolicy.mockResolvedValue({})
mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
metadata: { resourceVersion: '1' },
})
mockNetworkingApi.replaceNamespacedNetworkPolicy.mockResolvedValue({})
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})
mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
mockCustomApi.patchNamespacedCustomObject.mockResolvedValue({})
mockCustomApi.patchNamespacedCustomObjectStatus.mockResolvedValue({})
crashRecoveryMocks.evaluateCrashRecovery.mockReturnValue({
action: 'none',
message: 'Pod is healthy',
})
crashRecoveryMocks.evaluateCompletedRuntimePodRecovery.mockReturnValue({
action: 'replace',
message:
'mcp_host pod completed before workflow became terminal; creating replacement (attempt 1/3)',
newPhase: 'recovering',
newAttempt: 1,
})
crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
crashRecoveryMocks.isRecoverableContainerWaitingReason.mockImplementation(
(phase: string | undefined, reason: string | undefined) =>
Boolean(
reason &&
[
'CreateContainerConfigError',
'CreateContainerError',
'ImagePullBackOff',
'ErrImagePull',
'InvalidImageName',
'CrashLoopBackOff',
].includes(reason) &&
(phase === 'Pending' || (phase === 'Running' && reason === 'CrashLoopBackOff'))
)
)
runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockResolvedValue({
accessToken: 'runtime-access-token',
refreshToken: 'runtime-refresh-token',
mcpHostControlToken: 'mcp-host-control-token',
})
runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken.mockResolvedValue(
'mcp-host-control-token'
)
// Stub global.fetch used by cleanupRecipeArtifacts (best-effort).
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue({ ok: true, status: 204 } as unknown as Response)
)
})
it('stores inherited child mcp-host control credentials with the runtime-scope caller', async () => {
const reconciler = new WorkflowReconciler(deps)
runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockResolvedValueOnce({
accessToken: 'parent-runtime-access-token',
refreshToken: 'parent-runtime-refresh-token',
mcpHostControlToken: 'parent-workflow-control-token',
})
runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken.mockResolvedValueOnce(
'child-workflow-control-token'
)
await reconciler.ensureMcpHostRuntimeCredentials(
sandboxNamespace,
'child-run',
{ steps: [{ id: 'step-1', instruction: 'run with tools' }] },
'parent-recipe'
)
expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
sandboxNamespace,
'parent-recipe',
[]
)
expect(runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken).not.toHaveBeenCalled()
const createCall = mockCoreApi.createNamespacedSecret.mock.calls[0]?.[0] as {
body?: { data?: Record<string, string> }
}
expect(
Buffer.from(
createCall.body?.data?.['mcp-host-workflow-control-token'] ?? '',
'base64'
).toString('utf-8')
).toBe('parent-workflow-control-token')
})
it('mints read-scoped GFS credentials for recipe hosts by default', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-recipe', {
steps: [{ id: 'step-1', instruction: 'read inputs' }],
})
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'read-recipe',
{ scopes: ['gfs.read'] }
)
})
it('mints exactly read scope when a recipe mount is read-only', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-mount-recipe', {
steps: [{ id: 'step-1', instruction: 'read mounted inputs' }],
gfs: {
mounts: [{ drive: 'main', target: 'inputs', scopes: ['gfs.read'] }],
},
})
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'read-mount-recipe',
{ scopes: ['gfs.read'] }
)
})
it('keeps the read ceiling and adds write when a recipe mount is write-only', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'write-mount-recipe', {
steps: [{ id: 'step-1', instruction: 'write mounted outputs' }],
gfs: {
mounts: [{ drive: 'main', target: 'outputs', scopes: ['gfs.write'] }],
},
})
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'write-mount-recipe',
{ scopes: ['gfs.read', 'gfs.write'] }
)
})
it('orders and de-duplicates read-write mount scopes before minting', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-write-mount-recipe', {
steps: [{ id: 'step-1', instruction: 'read and write mounted content' }],
gfs: {
mounts: [
{
drive: 'main',
target: 'workspace',
scopes: ['gfs.write', 'gfs.read', 'gfs.write'],
},
],
},
})
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'read-write-mount-recipe',
{ scopes: ['gfs.read', 'gfs.write'] }
)
})
it('mints inherited child GFS scopes against the parent runtime identity', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(
sandboxNamespace,
'child-run',
{
steps: [{ id: 'step-1', instruction: 'use the parent grant' }],
gfs: {
mounts: [
{
drive: 'main',
target: 'workspace',
scopes: ['gfs.read', 'gfs.write'],
},
],
},
},
'parent-recipe'
)
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'parent-recipe',
{ scopes: ['gfs.read', 'gfs.write'] }
)
})
it('mints write-scoped GFS credentials when the recipe declares publish targets', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'publish-recipe', {
steps: [{ id: 'step-1', instruction: 'publish output' }],
gfs: {
publishTargets: [{ drive: 'main', target: 'outputs' }],
},
})
expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
sandboxNamespace,
'publish-recipe',
{ scopes: ['gfs.read', 'gfs.write'] }
)
})
it('deletes transport Services in mcp-server by label selector (agentic path, cross-namespace orphans)', async () => {
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'my-agentic-recipe'
await reconciler.reconcileDelete(recipeName, sandboxNamespace)
// The new cleanup call: mcp-server namespace + composite label selector
// matching what `buildService()` actually emits on Services (NOT `wrc`,
// which is only on NetworkPolicies built by workflowReconciler itself).
expect(mockCoreApi.deleteCollectionNamespacedService).toHaveBeenCalledTimes(1)
expect(mockCoreApi.deleteCollectionNamespacedService).toHaveBeenCalledWith({
namespace: mcpServerNamespace,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=workflow-recipes`,
})
})
it('still deletes per-mcp-server NetworkPolicies by label selector without deletecollection', async () => {
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'my-agentic-recipe'
mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(({ namespace }) =>
Promise.resolve({
items:
namespace === mcpServerNamespace
? [{ metadata: { name: `${recipeName}-dynamic-mcp-ingress` } }]
: [],
})
)
await reconciler.reconcileDelete(recipeName, sandboxNamespace)
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: mcpServerNamespace,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: `${recipeName}-dynamic-mcp-ingress`,
namespace: mcpServerNamespace,
})
})
it('deletes WRC-managed sandbox NetworkPolicies by recipe label for dynamic snippet policy names', async () => {
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'my-snippet-recipe'
mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(({ namespace }) =>
Promise.resolve({
items:
namespace === sandboxNamespace
? [{ metadata: { name: `${recipeName}-dynamic-snippet-ingress` } }]
: [],
})
)
await reconciler.reconcileDelete(recipeName, sandboxNamespace)
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: sandboxNamespace,
labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: `${recipeName}-dynamic-snippet-ingress`,
namespace: sandboxNamespace,
})
})
it('prefers artifact-reader HTTP artifact cleanup when the artifact-reader Service exists', async () => {
mockCoreApi.readNamespacedService.mockResolvedValue({})
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'agentic-recipe'
await reconciler.reconcileDelete(recipeName, sandboxNamespace)
expect(mockTokenFactory.signWrcArtifactDeleteToken).toHaveBeenCalledWith(
recipeName,
sandboxNamespace
)
expect(fetch).toHaveBeenCalledWith(
`http://wf-${recipeName}-artifact-reader.${sandboxNamespace}.svc.cluster.local:8080/api/v1/workflow/artifacts`,
expect.objectContaining({
method: 'DELETE',
headers: { authorization: 'Bearer stub-token' },
})
)
})
it('accepts snippet secretRef without requiring a matching spec.resources secret placeholder', () => {
const reconciler = new WorkflowReconciler(deps)
const error = reconciler.validateWorkflowSpec({
steps: [
{
id: 'fetch-price',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'coingecko_api_key',
secretRef: { name: 'coingecko-api', key: 'apiKey' },
},
],
},
},
},
],
})
expect(error).toBeUndefined()
})
it('rejects snippet secretRef entries that point at platform-managed workflow Secrets', () => {
const reconciler = new WorkflowReconciler(deps)
const error = reconciler.validateWorkflowSpec({
steps: [
{
id: 'read-runtime-token',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'runtime_token',
secretRef: { name: 'wf-example-coordinator-token', key: 'token' },
},
],
},
},
},
],
})
expect(error).toBe(
'step "read-runtime-token" cannot reference platform-managed secret "wf-example-coordinator-token"'
)
})
it('does not reject user-managed snippet Secrets only because their name mentions coordinator-token', () => {
const reconciler = new WorkflowReconciler(deps)
const error = reconciler.validateWorkflowSpec({
steps: [
{
id: 'read-api-key',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'api_key',
secretRef: { name: 'api-coordinator-token-rotator', key: 'token' },
},
],
},
},
},
],
})
expect(error).toBeUndefined()
})
it('fails workflow reconcile before pod creation when a snippet secretRef Secret is missing', async () => {
mockCoreApi.readNamespacedSecret.mockRejectedValueOnce({ code: 404 })
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile('snippet-recipe', 'uid-snippet', sandboxNamespace, {
steps: [
{
id: 'fetch-price',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'coingecko_api_key',
secretRef: { name: 'coingecko-api', key: 'apiKey' },
},
],
},
},
},
],
})
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: 'snippet secret "coingecko-api" was not found in namespace "sandbox-recipes"',
})
expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
name: 'coingecko-api',
namespace: sandboxNamespace,
})
})
it('resolves snippet resource ids through status.resourceInstances before validating access', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': 'snippet-recipe' } },
data: Object.fromEntries([['pass', 'present']]),
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'snippet-recipe',
'uid-snippet',
sandboxNamespace,
{
resources: [{ id: 'pg-auth', type: 'secret', data: { pass: 'redacted' } }],
steps: [
{
id: 'query-postgres',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [{ alias: 'pg', secretRef: { name: 'pg-auth', key: 'pass' } }],
},
},
},
],
},
{ resourceInstances: { 'pg-auth': 'snippet-recipe-pg-auth-d07f008b65ec' } }
)
expect(result.phase).toBe('deploying')
expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
name: 'snippet-recipe-pg-auth-d07f008b65ec',
namespace: sandboxNamespace,
})
})
it('preserves workflow status and signals skipStatusPatch on a transient infra error (no terminal failure)', async () => {
const reconciler = new WorkflowReconciler(deps)
// First call inside the reconcile try (getPodPhase) hits a transient API blip.
crashRecoveryMocks.getPodPhase.mockRejectedValueOnce(
Object.assign(new Error('connect ETIMEDOUT 10.96.0.1:443'), { code: 'ETIMEDOUT' })
)
const result = await reconciler.reconcile(
'flaky-workflow',
'uid-flaky',
sandboxNamespace,
{
steps: [
{ id: 'render', run: { type: 'snippet', language: 'typescript', code: 'return {}' } },
],
},
{ workflowExecution: { phase: 'recovering', message: 'prior transient blip' } }
)
// Must NOT latch the workflow to terminal failed.
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
// Preserves the current execution phase + message and tells WRC to skip the patch.
expect(result.skipStatusPatch).toBe(true)
expect(result.workflowPhase).toBe('recovering')
expect(result.message).toBe('prior transient blip')
// No status patch was written for the transient blip.
expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
it('fails workflow reconcile before pod creation when a snippet secretRef key is missing', async () => {
// Owned by the recipe (passes the Issue #637 ownership gate) but the key is
// absent — so the failure must be the key-missing message, not ownership.
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': 'snippet-recipe' } },
data: { otherKey: 'dmFsdWU=' },
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile('snippet-recipe', 'uid-snippet', sandboxNamespace, {
steps: [
{
id: 'fetch-price',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'coingecko_api_key',
secretRef: { name: 'coingecko-api', key: 'apiKey' },
},
],
},
},
},
],
})
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message:
'snippet secret key "apiKey" was not found in Secret "coingecko-api" in namespace "sandbox-recipes"',
})
})
it('fails workflow reconcile before pod creation when a snippet secretRef Secret is owned by another recipe (Issue #637)', async () => {
// The Secret EXISTS and HAS the requested key, but it is owned by a DIFFERENT
// recipe — the snippet runner pod must never be created with a foreign credential.
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': 'victim-recipe' } },
data: { apiKey: 'c3RvbGVu' },
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile('attacker-recipe', 'uid-attacker', sandboxNamespace, {
steps: [
{
id: 'fetch-price',
run: {
type: 'snippet',
language: 'typescript',
code: 'return { ok: true }',
capabilities: {
secrets: [
{
alias: 'coingecko_api_key',
secretRef: { name: 'coingecko-api', key: 'apiKey' },
},
],
},
},
},
],
})
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message:
'snippet secret "coingecko-api" is not accessible to recipe "attacker-recipe" — ' +
'it requires clerum.io/shared=true or clerum.io/owner-recipe=attacker-recipe',
})
// No snippet-runner pod was created for the denied recipe.
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
describe('snippet secret ownership on a triggered run (run-scoped child CR)', () => {
// A triggered run reconciles the run-scoped child CR `<parent>-<runid8>`, while
// WRC resolves the verified parent recipe as the runtime scope. Snippet Secret
// ownership must be judged against that parent scope, never the child name.
const childRecipeName = 'pr-review-abcd1234'
const parentScopeRecipeName = 'pr-review'
const triggeredRunId = 'run-id'
const scopeDenialMessage =
'snippet secret "github-token" is not accessible to recipe "pr-review" — ' +
'it requires clerum.io/shared=true or clerum.io/owner-recipe=pr-review'
function snippetSecretSpec(secretName = 'github-token') {
return {
steps: [
{
id: 'triage',
run: {
type: 'snippet' as const,
language: 'typescript' as const,
code: 'return { ok: true }',
capabilities: {
secrets: [{ alias: 'github_token', secretRef: { name: secretName, key: 'token' } }],
},
},
},
],
}
}
function reconcileTriggeredChild(
spec = snippetSecretSpec(),
currentStatus?: { resourceInstances?: Record<string, string> }
) {
// Settle the output gates so a pass that clears the preflight reaches runtime
// pod creation. The two names differ in scope ON PURPOSE: the output claim and
// its anchor pod belong to the PARENT scope (`pr-review-workflow-output-anchor`),
// while the prepare pod is RUN-scoped (`pr-review-abcd1234-workflow-output-prepare`).
// Without these, the pass stops at "Created workflow output anchor pod" and
// the snippet-runner pod is never created.
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === `${parentScopeRecipeName}-workflow-output-anchor`) return 'Running'
if (name === `${childRecipeName}-workflow-output-prepare`) return 'Succeeded'
return undefined
})
return new WorkflowReconciler(deps).reconcile(
childRecipeName,
'uid-pr-review-child',
sandboxNamespace,
spec,
currentStatus,
undefined,
parentScopeRecipeName,
triggeredRunId
)
}
function createdPodNames(): string[] {
return mockCoreApi.createNamespacedPod.mock.calls.map(call => call[0].body.metadata.name)
}
it('creates the snippet runner when the Secret is owned by the parent scope recipe', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': parentScopeRecipeName } },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
name: 'github-token',
namespace: sandboxNamespace,
})
expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
const runnerPod = mockCoreApi.createNamespacedPod.mock.calls
.map(call => call[0].body)
.find(body => body.metadata.name === `${childRecipeName}-snippet-runner`)
expect(runnerPod).toBeDefined()
const projectedSecretRefs = runnerPod.spec.containers.flatMap(
(container: { env?: Array<{ valueFrom?: { secretKeyRef?: unknown } }> }) =>
(container.env ?? []).flatMap(env =>
env.valueFrom?.secretKeyRef ? [env.valueFrom.secretKeyRef] : []
)
)
expect(projectedSecretRefs).toEqual([{ name: 'github-token', key: 'token' }])
})
it('fails before pod creation when the Secret is owned by another recipe, naming the parent scope', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': 'other-recipe' } },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: scopeDenialMessage,
})
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('fails before pod creation when the Secret is owned by the run-scoped child name', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': childRecipeName } },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: scopeDenialMessage,
})
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('fails closed when the Secret carries both owner-recipe=<parent> and shared=true', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: {
labels: {
'clerum.io/owner-recipe': parentScopeRecipeName,
'clerum.io/shared': 'true',
},
},
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: scopeDenialMessage,
})
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('fails closed when the Secret carries no ownership label', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: {} },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
message: scopeDenialMessage,
})
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
it('keeps shared=true Secrets accessible to the triggered run', async () => {
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/shared': 'true' } },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild()
expect(result.phase).not.toBe('failed')
expect(result.workflowPhase).not.toBe('failed')
expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
})
it('judges a Secret resolved through a supplied resourceInstances map against the parent scope', async () => {
// Pins only the preflight: given a resourceInstances map, the snippet resource id
// resolves through it and ownership is judged against the parent scope. The map
// is supplied by hand; how a triggered child builds it is not covered here.
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
metadata: { labels: { 'clerum.io/owner-recipe': parentScopeRecipeName } },
data: { token: 'fixture-token' },
})
const result = await reconcileTriggeredChild(
{
...snippetSecretSpec('gh-auth'),
resources: [{ id: 'gh-auth', type: 'secret' as const, data: { token: 'fixture-token' } }],
},
{ resourceInstances: { 'gh-auth': 'pr-review-gh-auth-d07f008b65ec' } }
)
expect(result.phase).not.toBe('failed')
expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
name: 'pr-review-gh-auth-d07f008b65ec',
namespace: sandboxNamespace,
})
expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
})
it('still fails closed (throws) when Secret ownership cannot be read', async () => {
mockCoreApi.readNamespacedSecret.mockRejectedValueOnce({ code: 503 })
await expect(reconcileTriggeredChild()).rejects.toThrow(
'snippet secret "github-token" ownership could not be verified in namespace "sandbox-recipes"'
)
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
})
it('skips artifact HTTP cleanup when neither artifact-reader nor mcp-host Service exists', async () => {
mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
const reconciler = new WorkflowReconciler(deps)
const recipeName = 'pure-custom-recipe'
await reconciler.reconcileDelete(recipeName, sandboxNamespace)
expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
name: `wf-${recipeName}-artifact-reader`,
namespace: sandboxNamespace,
})
expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
name: `wf-${recipeName}-mcp-host`,
namespace: sandboxNamespace,
})
expect(mockTokenFactory.signWrcArtifactDeleteToken).not.toHaveBeenCalled()
expect(fetch).not.toHaveBeenCalled()
expect(mockCoreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
name: `${recipeName}-workflow-output`,
namespace: sandboxNamespace,
})
})
it('gates coordinator creation while the mcp-host pod is still waiting for scheduling', async () => {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
return undefined
})
crashRecoveryMocks.getPodReadiness.mockResolvedValueOnce({
ready: false,
phase: 'Pending',
schedulingReason: 'Unschedulable',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing' } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'deploying',
workflowPhase: 'initializing',
})
expect(result.message).toContain('Waiting for mcp-host pod to become Ready')
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('agentic-recipe-mcp-host')
expect(podNames).not.toContain('agentic-recipe-coordinator')
})
it('preserves required toolChoice in the workflow config ConfigMap', async () => {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'tool-choice-recipe-workflow-output-anchor') return 'Running'
if (name === 'tool-choice-recipe-workflow-output-prepare') return 'Succeeded'
return undefined
})
mockMcpHostReadiness({ ready: false, phase: 'Pending' })
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'tool-choice-recipe',
'uid-tool-choice',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [
{
id: 'use-gfs',
instruction: 'Use the available GFS tools',
toolChoice: 'required',
},
],
},
{ workflowExecution: { phase: 'initializing' } },
undefined,
'tool-choice-recipe',
'run-1'
)
const configMap = mockCoreApi.createNamespacedConfigMap.mock.calls
.map(call => call[0].body)
.find(body => body.metadata?.name === 'tool-choice-recipe-workflow-config')
const workflowConfig = JSON.parse(configMap.data['config.json'])
expect(workflowConfig.steps[0]).toMatchObject({
id: 'use-gfs',
toolChoice: 'required',
})
})
it('does not add toolChoice to workflow config steps when it is absent', async () => {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'default-tool-choice-recipe-workflow-output-anchor') return 'Running'
if (name === 'default-tool-choice-recipe-workflow-output-prepare') return 'Succeeded'
return undefined
})
mockMcpHostReadiness({ ready: false, phase: 'Pending' })
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'default-tool-choice-recipe',
'uid-default-tool-choice',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'answer', instruction: 'Answer without a tool requirement' }],
},
{ workflowExecution: { phase: 'initializing' } },
undefined,
'default-tool-choice-recipe',
'run-1'
)
const configMap = mockCoreApi.createNamespacedConfigMap.mock.calls
.map(call => call[0].body)
.find(body => body.metadata?.name === 'default-tool-choice-recipe-workflow-config')
const workflowConfig = JSON.parse(configMap.data['config.json'])
expect(workflowConfig.steps[0]).not.toHaveProperty('toolChoice')
})
it('preserves recovering while the mcp-host pod is not ready', async () => {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
return undefined
})
crashRecoveryMocks.getPodReadiness.mockResolvedValueOnce({
ready: false,
phase: 'Pending',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'recovering', attempt: 1 } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'deploying',
workflowPhase: 'recovering',
})
})
it('fails stale pre-coordinator readiness wait for a Running non-ready mcp-host pod', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Running' })
mockMcpHostReadiness({
ready: false,
phase: 'Running',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(10 * 60_000) } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
})
expect(result.message).toContain('Waiting for mcp-host pod to become Ready (phase=Running)')
expect(result.message).toContain('readiness deadline exceeded')
})
it('fails stale pre-coordinator readiness wait for a Pending non-recoverable mcp-host pod', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Pending' })
mockMcpHostReadiness({
ready: false,
phase: 'Pending',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(10 * 60_000) } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'failed',
workflowPhase: 'failed',
})
expect(result.message).toContain('Waiting for mcp-host pod to become Ready (phase=Pending)')
expect(result.message).toContain('readiness deadline exceeded')
})
it('keeps waiting when mcp-host readiness wait is still inside the deadline', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Running' })
mockMcpHostReadiness({
ready: false,
phase: 'Running',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(60_000) } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'deploying',
workflowPhase: 'initializing',
})
expect(result.message).not.toContain('readiness deadline exceeded')
})
it('does not fail the first mcp-host readiness wait before startedAt is persisted', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Pending' })
mockMcpHostReadiness({
ready: false,
phase: 'Pending',
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing' } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'deploying',
workflowPhase: 'initializing',
})
expect(result.message).not.toContain('readiness deadline exceeded')
})
it('does not gate an already running workflow on transient mcp-host readiness', async () => {
crashRecoveryMocks.getPodPhase
.mockResolvedValueOnce('Running')
.mockResolvedValueOnce('Running')
.mockResolvedValueOnce('Succeeded')
.mockResolvedValueOnce(undefined)
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'running', attempt: 0 } },
undefined,
'agentic-recipe',
'run-1'
)
expect(crashRecoveryMocks.getPodReadiness).not.toHaveBeenCalled()
expect(result).toMatchObject({
phase: 'active',
workflowPhase: 'running',
})
})
it('does not replace a completed mcp-host after workflow completion', async () => {
crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
if (name === 'agentic-recipe-coordinator') return 'Succeeded'
if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
if (name === 'agentic-recipe-mcp-host') return 'Succeeded'
return undefined
})
const reconciler = new WorkflowReconciler(deps)
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'completed', attempt: 0 } },
undefined,
'agentic-recipe',
'run-1'
)
expect(crashRecoveryMocks.evaluateCompletedRuntimePodRecovery).not.toHaveBeenCalled()
expect(crashRecoveryMocks.getPodReadiness).not.toHaveBeenCalled()
expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()
expect(result).toMatchObject({
phase: 'active',
workflowPhase: 'completed',
})
})
it('creates the coordinator on a later reconcile after mcp-host becomes Ready', async () => {
let mcpHostPhaseReads = 0
mockAgenticOutputPodPhases({
mcpHost: () => {
mcpHostPhaseReads += 1
return mcpHostPhaseReads === 1 ? undefined : 'Running'
},
})
mockMcpHostReadinessSequence(
{ ready: false, phase: 'Pending' },
{ ready: true, phase: 'Running' }
)
const reconciler = new WorkflowReconciler(deps)
const spec = {
agent: { provider: 'openai' as const, model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
}
await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
spec,
{ workflowExecution: { phase: 'initializing' } },
undefined,
'agentic-recipe',
'run-1'
)
mockCoreApi.createNamespacedPod.mockClear()
const result = await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
spec,
{ workflowExecution: { phase: 'initializing' } },
undefined,
'agentic-recipe',
'run-1'
)
expect(result).toMatchObject({
phase: 'deploying',
workflowPhase: 'initializing',
})
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('agentic-recipe-coordinator')
const createdPods = mockCoreApi.createNamespacedPod.mock.calls.map(call => call[0].body)
const artifactReaderPod = createdPods.find(
pod => pod.metadata.name === 'agentic-recipe-artifact-reader'
)
const coordinatorPod = createdPods.find(
pod => pod.metadata.name === 'agentic-recipe-coordinator'
)
expect(
artifactReaderPod.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]
.labelSelector.matchExpressions
).toContainEqual({
key: 'clerum.io/component',
operator: 'In',
values: ['workflow-output-anchor'],
})
expect(
coordinatorPod.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]
.labelSelector.matchExpressions
).toContainEqual({
key: 'clerum.io/component',
operator: 'In',
values: ['workflow-output-anchor'],
})
})
it('replaces a CrashLoopBackOff mcp-host pod before waiting on readiness', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Running' })
mockMcpHostReadiness({ ready: true, phase: 'Running' })
crashRecoveryMocks.getContainerWaitingReason.mockImplementation(async (_api, name: string) =>
name === 'agentic-recipe-mcp-host' ? 'CrashLoopBackOff' : undefined
)
crashRecoveryMocks.evaluateCrashRecovery.mockReturnValueOnce({
action: 'replace',
message: 'Pod in Running with CrashLoopBackOff; creating replacement (attempt 1/3)',
newPhase: 'recovering',
newAttempt: 1,
})
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing', attempt: 0 } },
undefined,
'agentic-recipe',
'run-1'
)
expect(crashRecoveryMocks.evaluateCrashRecovery).toHaveBeenCalledWith(
'Running',
{ phase: 'initializing', attempt: 0 },
'CrashLoopBackOff'
)
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
mockCoreApi,
'agentic-recipe-mcp-host',
sandboxNamespace
)
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('agentic-recipe-mcp-host')
expect(podNames).toContain('agentic-recipe-coordinator')
})
it('replaces a completed mcp-host pod before waiting on readiness', async () => {
mockAgenticOutputPodPhases({ mcpHost: 'Succeeded' })
mockMcpHostReadiness({ ready: true, phase: 'Running' })
const reconciler = new WorkflowReconciler(deps)
await reconciler.reconcile(
'agentic-recipe',
'uid-agentic',
sandboxNamespace,
{
agent: { provider: 'openai', model: 'gpt-4.1' },
steps: [{ id: 'brief', instruction: 'write the brief' }],
},
{ workflowExecution: { phase: 'initializing', attempt: 0 } },
undefined,
'agentic-recipe',
'run-1'
)
expect(crashRecoveryMocks.evaluateCompletedRuntimePodRecovery).toHaveBeenCalledWith(
{ phase: 'initializing', attempt: 0 },
'mcp_host'
)
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
mockCoreApi,
'agentic-recipe-mcp-host',
sandboxNamespace
)
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('agentic-recipe-mcp-host')
expect(podNames).toContain('agentic-recipe-coordinator')
})
describe('teardownComputePodsForTerminalRun', () => {
it('deletes coordinator, mcp-host, and snippet-runner but PRESERVES artifact-reader', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
const deletedPods = crashRecoveryMocks.deletePodIfExists.mock.calls.map(
call => call[1] as string
)
expect(deletedPods).toContain('wf-run-7-coordinator')
expect(deletedPods).toContain('wf-run-7-mcp-host')
expect(deletedPods).toContain('wf-run-7-snippet-runner')
// Artifacts must stay downloadable — artifact-reader is never torn down here.
expect(deletedPods).not.toContain('wf-run-7-artifact-reader')
// All deletions target the sandbox namespace.
for (const call of crashRecoveryMocks.deletePodIfExists.mock.calls) {
expect(call[2]).toBe(sandboxNamespace)
}
expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()
})
it('is DELETE-first and 404-safe even when pods are already gone', async () => {
crashRecoveryMocks.deletePodIfExists.mockResolvedValue(undefined)
const reconciler = new WorkflowReconciler(deps)
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
// Witness: the teardown ran (three DELETEs) and none of them was
// preceded by a raw pod GET, which getPodPresence alone would not show.
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)
expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()
expect(mockCoreApi.readNamespacedPod).not.toHaveBeenCalled()
})
it('is idempotent: a second pass still DELETE-first (404-safe)', async () => {
const reconciler = new WorkflowReconciler(deps)
await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
crashRecoveryMocks.deletePodIfExists.mockClear()
await expect(
reconciler.teardownComputePodsForTerminalRun('wf-run-7')
).resolves.toBeUndefined()
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)
})
})
})
describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {
const sandboxNamespace = 'sandbox-recipes'
const mcpServerNamespace = 'mcp-server'
const mockCoreApi = {
readNamespacedPod: vi.fn().mockResolvedValue({}),
readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
readNamespacedService: vi.fn().mockResolvedValue({}),
readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
createNamespacedSecret: vi.fn().mockResolvedValue({}),
createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
createNamespacedService: vi.fn().mockResolvedValue({}),
createNamespacedPod: vi.fn().mockResolvedValue({}),
patchNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedPod: vi.fn().mockResolvedValue({}),
deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
deleteNamespacedService: vi.fn().mockResolvedValue({}),
}
const mockCustomApi = {
patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
}
const mockNetworkingApi = {
createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
}
const mockModelConfigHandler = {
configurePluginWorkloadSdkBootstrap: vi.fn().mockResolvedValue({
status: 202,
body: {
configured: true,
ready: true,
provider: 'zai',
model: 'glm-4.7',
contractVersion: 2,
policyRevision: 1,
policyHash: 'a'.repeat(64),
defaultTargetRef: 'primary-zai',
defaultProvider: 'zai',
defaultModel: 'glm-4.7',
clientNotificationsPolicyReady: true,
clientNotificationsPolicyState: 'active',
},
}),
}
const mockTokenFactory = {
signWrcConfigureToken: vi.fn().mockResolvedValue('wrc-configure-token'),
signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('stub-token'),
}
const mockPluginWorkloadSdkRevocationClient = {
revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),
finalize: vi.fn(),
}
const makeDeps = () =>
({
coreApi: mockCoreApi,
customApi: mockCustomApi,
networkingApi: mockNetworkingApi,
config: {
coordinatorImage: 'coordinator:test',
mcpHostImage: 'mcp-host:test',
wrcEndpoint: 'http://wrc.example/api',
sandboxNamespace,
mcpServerNamespace,
imagePullPolicy: 'IfNotPresent' as const,
maxWorkflowSteps: 100,
runtimeTokenTtlSeconds: 3600,
runtimeTokenRefreshBeforeSeconds: 300,
pluginWorkloadSdkEnabled: true,
},
tokenFactory: mockTokenFactory,
modelConfigHandler: mockModelConfigHandler,
pluginWorkloadSdkRevocationClient: mockPluginWorkloadSdkRevocationClient,
}) as unknown as WorkflowReconcilerDeps
const sdkBootstrapProof = () => ({
configured: true,
ready: true,
provider: 'zai',
model: 'glm-4.7',
contractVersion: 2,
policyRevision: 1,
policyHash: 'a'.repeat(64),
defaultTargetRef: 'primary-zai',
defaultProvider: 'zai',
defaultModel: 'glm-4.7',
clientNotificationsPolicyReady: true,
clientNotificationsPolicyState: 'active',
})
beforeEach(() => {
vi.clearAllMocks()
crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
crashRecoveryMocks.evaluateCrashRecovery.mockReturnValue({ action: 'none', message: 'healthy' })
mockCoreApi.createNamespacedPod.mockResolvedValue({})
mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
mockCoreApi.readNamespacedEndpoints.mockRejectedValue({ code: 404 })
mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
// clearAllMocks keeps implementations, so a DELETE failure one test
// injects must not leak into the next teardown.
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: sdkBootstrapProof(),
})
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue({ ok: true, status: 204 } as unknown as Response)
)
})
const sdkSpec = (overrides = {}) =>
({
agent: { provider: 'zai', model: 'glm-4.7' },
steps: [{ id: 'keepalive', instruction: 'ack' }],
workloads: [{ id: 'sdk-caller', type: 'deployment', image: 'caller:test' }],
pluginWorkloadSdk: {
promptBridge: {},
allowedCallers: ['sdk-caller'],
},
...overrides,
}) as unknown as Parameters<WorkflowReconciler['reconcile']>[3]
it('rotates access, refresh, and control tokens when runtime scopes drift', async () => {
const now = Math.floor(Date.now() / 1000)
const staleBinding = {
exp: now + 3600,
recipeNamespace: sandboxNamespace,
recipeName: 'sdk-recipe',
hostRefs: [`${sandboxNamespace}/sdk-recipe`],
scopes: [],
}
const staleGfsBinding = {
...staleBinding,
hostRefs: [`host:3rd:${sandboxNamespace}/sdk-recipe`],
scopes: ['gfs.read'],
}
mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
data: {
'mcp-host-runtime-access-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
'base64'
),
'mcp-host-runtime-refresh-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
'base64'
),
'mcp-host-workflow-control-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
'base64'
),
'mcp-host-gfs-token': Buffer.from(unsignedRuntimeJwt(staleGfsBinding)).toString('base64'),
},
})
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'sdk-recipe', sdkSpec())
expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
sandboxNamespace,
'sdk-recipe',
['plugin-workload-sdk']
)
expect(mockCoreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)
const patch = mockCoreApi.patchNamespacedSecret.mock.calls[0]?.[0] as {
body?: { data?: Record<string, string> }
}
expect(patch.body?.data).toEqual(
expect.objectContaining({
'mcp-host-runtime-access-token': expect.any(String),
'mcp-host-runtime-refresh-token': expect.any(String),
'mcp-host-workflow-control-token': expect.any(String),
})
)
})
it('creates the mcp-host but NOT the coordinator when no run is triggered', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined // no run id → awaitsTriggeredRun
)
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('sdk-recipe-mcp-host')
expect(podNames).not.toContain('sdk-recipe-coordinator')
// The eager mcp-host must get the same runtime-token + SDK-token Secrets the
// triggered-run path provisions, or it boots NotReady (degraded runtime auth).
const secretNames = mockCoreApi.createNamespacedSecret.mock.calls.map(
call => call[0].body.metadata.name
)
expect(secretNames).toContain('wf-sdk-recipe-mcp-host-runtime-tokens')
const runtimeSecret = mockCoreApi.createNamespacedSecret.mock.calls
.map(call => call[0].body)
.find(secret => secret.metadata.name === 'wf-sdk-recipe-mcp-host-runtime-tokens')
expect(runtimeSecret?.stringData?.[['mcp-host-gfs', 'token'].join('-')]).toBe(
'gfs-runtime-value'
)
// The SDK lane NetworkPolicies must also come up so the sdk-caller can
// reach the SDK server on :8099.
const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
call => call[0].body.metadata.name
)
// A stepless SDK host has no coordinator. The eager network lane must not
// create workflow-only coordinator policies that would later be orphaned.
expect(npNames).not.toContain('sdk-recipe-coord-to-mcp-host')
expect(npNames).not.toContain('sdk-recipe-coord-to-mcp-host-ingress')
expect(npNames).not.toContain('sdk-recipe-coord-to-wrc')
expect(npNames).toContain('sdk-recipe-workload-to-mcp-host-sdk-ingress')
expect(npNames).toContain('sdk-recipe-workload-to-mcp-host-sdk-egress')
expect(npNames).toContain('sdk-recipe-mcp-host-to-gfs')
// The mcp-host headless Service must exist so the SDK endpoint DNS resolves.
const svcNames = mockCoreApi.createNamespacedService.mock.calls.map(
call => call[0].body.metadata.name
)
expect(svcNames).toContain('wf-sdk-recipe-mcp-host')
})
it('leaves coordinator GFS NetworkPolicy ownership to the outer recipe reconciler', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec({ gfs: { publishTargets: [{ drive: 'main', target: 'outputs' }] } }),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
call => call[0].body.metadata.name
)
expect(npNames).not.toContain('sdk-recipe-coordinator-to-gfs')
})
it('creates eager mcp-host for SDK-only recipes without agentic steps', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-only',
'uid-sdk-only',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('sdk-only-mcp-host')
expect(podNames).not.toContain('sdk-only-coordinator')
const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
call => call[0].body.metadata.name
)
expect(npNames).not.toContain('sdk-only-coord-to-mcp-host')
expect(npNames).not.toContain('sdk-only-coord-to-mcp-host-ingress')
expect(npNames).not.toContain('sdk-only-coord-to-wrc')
expect(result.phase).toBe('active')
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
'zai',
'glm-4.7',
expect.any(String),
'wrc-configure-token',
'promptBridge',
null
)
})
it('reads the eager mcp-host Service and sends no POST when the live one matches', async () => {
const serviceName = buildMcpHostServiceName('sdk-only')
const live = {
...buildMcpHostHeadlessService('sdk-only', sandboxNamespace),
metadata: {
...buildMcpHostHeadlessService('sdk-only', sandboxNamespace).metadata,
resourceVersion: 'rv-3',
},
}
mockCoreApi.readNamespacedService.mockImplementation(async ({ name }: { name: string }) => {
if (name === serviceName) return structuredClone(live)
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-only',
'uid-sdk-only',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
// This fixture has no replaceNamespacedService, so a replace would throw
// and the phase would not be 'active'.
expect(result.phase).toBe('active')
expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
name: serviceName,
namespace: sandboxNamespace,
})
const createdServiceNames = mockCoreApi.createNamespacedService.mock.calls.map(
call => call[0].body.metadata.name
)
expect(createdServiceNames).not.toContain(serviceName)
})
// The disabled branch applies no policy, so it must not report a summary: an
// empty summary reads as "evaluated, no conflict" and would clear a published
// WorkflowNetworkPolicyOwnership condition.
it('reports no NetworkPolicy summary when the SDK runtime is disabled', async () => {
const deps = makeDeps()
const reconciler = new WorkflowReconciler({
...deps,
config: { ...deps.config, pluginWorkloadSdkEnabled: false },
})
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-only',
'uid-sdk-only',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
expect(result.phase).toBe('active')
expect(result.message).toBe('Plugin Workload SDK runtime disabled')
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(result.networkPolicies).toBeUndefined()
})
describe('NetworkPolicy ownership conflicts', () => {
const foreignOwnedPolicy = (name: string) => ({
metadata: {
name,
namespace: sandboxNamespace,
resourceVersion: '7',
ownerReferences: [
{
apiVersion: 'apps/v1',
kind: 'Deployment',
name: 'another-controller',
uid: 'foreign-owner-uid',
controller: true,
},
],
},
spec: { podSelector: {} },
})
const foreignOnly = (foreignName: string) =>
mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name: string }) => {
if (name === foreignName) return foreignOwnedPolicy(foreignName)
throw { code: 404 }
}
)
const writesTo = (name: string) => [
...mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.filter(
call => call[0].body.metadata.name === name
),
...mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
call => call[0].name === name
),
]
const createdNames = () =>
mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
call => call[0].body.metadata.name
)
const ownershipCondition = (conditions: unknown) =>
(conditions as Array<{ type: string }> | undefined)?.find(
condition => condition.type === 'WorkflowNetworkPolicyOwnership'
)
it('SDK-only lane returns the conflict and leaves the foreign policy untouched', async () => {
const foreignName = 'sdk-only-workload-to-mcp-host-sdk-ingress'
foreignOnly(foreignName)
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-only',
'uid-sdk-only',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
expect.objectContaining({ name: foreignName })
)
// The conflict did not stop the rest of the pass.
expect(createdNames()).toContain('sdk-only-workload-to-mcp-host-sdk-egress')
expect(result.phase).toBe('active')
// The eager apply prunes too, so its summary carries the prune and legacy facts.
expect(result.networkPolicies).toEqual({
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: false,
prune: 'converged',
legacy: 'removed',
})
expect(writesTo(foreignName)).toHaveLength(0)
})
// Every eager host status that is not `ready` still ran the apply when it
// carries a summary, so the SDK-only lane must hand that summary on: dropping
// it would read as "returned before applying" and keep a stale condition.
it.each([
{ status: 'awaiting_policy', phase: 'awaiting_policy' },
{ status: 'deploying', phase: 'deploying' },
{ status: 'provider_unavailable', phase: 'provider_unavailable' },
{ status: 'failed', phase: 'failed' },
] as const)(
'SDK-only lane passes the eager apply summary through on $status',
async ({ status, phase }) => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: {
ensureEagerSdkMcpHost: (...args: unknown[]) => unknown
}
}
).pluginWorkloadSdkProvisioner
const foreignName = 'sdk-only-workload-to-mcp-host-sdk-ingress'
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status,
networkPolicies: {
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: true,
prune: 'converged',
legacy: 'removed',
},
})
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-only',
'uid-sdk-only',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(ensure.mock.calls[0]?.[0]).toBe('sdk-only')
expect(result.phase).toBe(phase)
expect(result.networkPolicies).toEqual({
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: true,
prune: 'converged',
legacy: 'removed',
})
}
)
it('workflow lane surfaces the conflict as a False condition and clears it on recovery', async () => {
const foreignName = 'sdk-recipe-workload-to-mcp-host-sdk-ingress'
foreignOnly(foreignName)
const reconciler = new WorkflowReconciler(makeDeps())
const first = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
expect.objectContaining({ name: foreignName })
)
expect(createdNames()).toContain('sdk-recipe-workload-to-mcp-host-sdk-egress')
expect(first.phase).not.toBe('failed')
expect(first.workflowPhase).not.toBe('failed')
expect(first.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message: expect.stringContaining(foreignName),
}),
])
// The condition travels only in its own field, never with the
// workflow-output conditions that patchStatus merges separately.
expect(ownershipCondition(first.workflowConditions)).toBeUndefined()
expect(writesTo(foreignName)).toHaveLength(0)
// The foreign owner goes away: the policy is created and the owned set
// is empty, so patchStatus removes the condition.
mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
const second = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{
workflowExecution: { phase: 'initializing' },
conditions: first.networkPolicyOwnershipConditions,
},
undefined,
'sdk-recipe',
undefined
)
expect(createdNames()).toContain(foreignName)
expect(second.networkPolicyOwnershipConditions).toEqual([])
})
// The pass never reached the apply, so it cannot say whether the conflict is
// gone. The field stays undefined and patchStatus keeps the published one.
it('workflow lane leaves the ownership field undefined when the eager host fails before applying', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
}
).pluginWorkloadSdkProvisioner
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status: 'failed',
})
const existing = {
type: 'WorkflowNetworkPolicyOwnership',
status: 'False' as const,
reason: 'OwnershipConflict',
message: 'sdk-recipe-mcp-host-to-gfs (owner-reference-mismatch)',
lastTransitionTime: '2026-09-23T10:00:00.000Z',
}
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' }, conditions: [existing] },
undefined,
'sdk-recipe',
undefined
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(result.phase).toBe('failed')
expect(result).not.toHaveProperty('networkPolicyOwnershipConditions')
expect(ownershipCondition(result.workflowConditions)).toBeUndefined()
})
// The pod failed after the policies were applied, so the summary is a real
// evaluation and the conflict it found is published with the failure.
it('workflow lane publishes the evaluated set when the eager host fails after applying', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
}
).pluginWorkloadSdkProvisioner
const foreignName = 'sdk-recipe-mcp-host-to-gfs'
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status: 'failed',
networkPolicies: {
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: false,
prune: 'converged',
legacy: 'removed',
},
})
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(result.phase).toBe('failed')
expect(result.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message: expect.stringContaining(`${foreignName} (owner-reference-mismatch)`),
}),
])
})
// The early return skips the run-lane apply, so the eager summary is the
// only source of the retry flag for this pass.
it('workflow lane asks for a retry when the eager host fails after a deferred policy', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
}
).pluginWorkloadSdkProvisioner
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status: 'failed',
networkPolicies: {
conflicts: [],
retryPending: true,
prune: 'converged',
legacy: 'removed',
},
})
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(result.phase).toBe('failed')
expect(result.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'RetryPending',
}),
])
expect(result.networkPolicyRetryPending).toBe(true)
})
// provider_unavailable overrides workflowConditions after the spread, so it
// must still carry the ownership set and the retry flag from the eager apply.
it('workflow lane publishes the eager apply summary on provider_unavailable', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
}
).pluginWorkloadSdkProvisioner
const foreignName = 'sdk-recipe-mcp-host-to-gfs'
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status: 'provider_unavailable',
networkPolicies: {
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: true,
prune: 'converged',
legacy: 'removed',
},
})
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(result.phase).toBe('active')
expect(result.workflowConditions).toEqual(
expect.arrayContaining([
expect.objectContaining({ type: 'PluginWorkloadSdkProviderUnavailable' }),
])
)
expect(result.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message: expect.stringContaining(`${foreignName} (owner-reference-mismatch)`),
}),
expect.objectContaining({
type: 'WorkflowNetworkPoliciesConverged',
status: 'False',
reason: 'RetryPending',
}),
])
expect(result.networkPolicyRetryPending).toBe(true)
})
// A conflict that persists across passes keeps the time it was first seen,
// so the condition does not look like a new transition on every pass.
it('workflow lane keeps the first transition time of a persisting conflict', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const provisioner = (
reconciler as unknown as {
pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
}
).pluginWorkloadSdkProvisioner
const foreignName = 'sdk-recipe-mcp-host-to-gfs'
const firstSeen = '2026-09-23T10:00:00.000Z'
const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
status: 'failed',
networkPolicies: {
conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
retryPending: false,
prune: 'converged',
legacy: 'removed',
},
})
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{
workflowExecution: { phase: 'initializing' },
conditions: [
{
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
reason: 'OwnershipConflict',
message: `NetworkPolicy ownership conflict: ${foreignName} (owner-reference-mismatch)`,
lastTransitionTime: firstSeen,
},
],
},
undefined,
'sdk-recipe',
undefined
)
expect(ensure).toHaveBeenCalledTimes(1)
expect(result.networkPolicyOwnershipConditions).toEqual([
expect.objectContaining({
type: 'WorkflowNetworkPolicyOwnership',
status: 'False',
message: expect.stringContaining(foreignName),
lastTransitionTime: firstSeen,
}),
])
})
})
it('creates a provider-free eager mcp-host for clientNotifications-only without an agent', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: {
configured: true,
ready: true,
contractVersion: 2,
capabilityFamily: 'clientNotifications',
policyReady: true,
policyState: 'active',
},
})
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'notifications-only',
'uid-notifications-only',
sandboxNamespace,
sdkSpec({
agent: undefined,
steps: undefined,
pluginWorkloadSdk: {
clientNotifications: { allowedEventTypes: ['e2e.test'] },
allowedCallers: ['sdk-caller'],
},
})
)
expect(result.phase).toBe('active')
const pod = mockCoreApi.createNamespacedPod.mock.calls[0]?.[0].body
const envNames = (pod.spec.containers[0].env ?? []).map((entry: { name: string }) => entry.name)
expect(envNames).not.toContain('CLERUM_MODEL_PROVIDER')
expect(envNames).not.toContain('CLERUM_MODEL')
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
undefined,
undefined,
expect.any(String),
'wrc-configure-token',
'clientNotifications'
)
})
it('cleans SDK-only host resources idempotently without creating workflow resources', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
expect(mockCoreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'wf-sdk-only-mcp-host',
namespace: sandboxNamespace,
})
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'wf-sdk-only-plugin-workload-sdk-token',
namespace: sandboxNamespace,
})
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'wf-sdk-only-mcp-host-runtime-tokens',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'sdk-only-workload-to-mcp-host-sdk-ingress',
namespace: sandboxNamespace,
})
expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
})
// R4-L3: the SDK-only teardown is what makes the `unmanaged` marker writer
// true. The legacy mcp-servers internet policy may carry no labels, so the
// label sweep cannot be trusted to remove it; the teardown deletes it by name.
it('R4-L3: deletes the legacy internet policy by name in the SDK-only teardown, next to the label sweep', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
// Liveness witness: the label sweep ran in both namespaces.
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: sandboxNamespace,
labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
})
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: mcpServerNamespace,
labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'sdk-only-mcp-servers-egress-internet',
namespace: mcpServerNamespace,
})
})
it('fails live SDK cleanup with the remaining shared legacy-delete window', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name: string }) => {
if (name === 'sdk-only-mcp-servers-egress-internet') {
throw { code: 403, message: 'forbidden' }
}
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps())
try {
const pending = await reconciler
.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
.catch(error => error)
expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(
LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS
)
// Positive API witness: the shared legacy path attempted the DELETE;
// the typed error describes the live cleanup's remaining backoff.
expect(
mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
)
).toHaveLength(1)
} finally {
vi.useRealTimers()
}
})
// A-1: a pending legacy mcp-servers internet DELETE must not hold the SDK
// host pod and token Secrets in place. The legacy policy is unrelated to the
// SDK credentials, so the pod and Secret deletes run on every pass while the
// cleanup still fails closed with the legacy-pending error.
it('A-1: deletes the SDK host pod and token Secrets while the legacy policy DELETE is pending', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name: string }) => {
if (name === 'sdk-only-mcp-servers-egress-internet') {
throw { code: 403, message: 'forbidden' }
}
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps())
const policyDeletes = (name: string) =>
mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === name
)
try {
const first = await reconciler
.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
.catch(error => error)
const second = await reconciler
.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
.catch(error => error)
expect(first).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
expect(second).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
// Liveness witness: both passes ran the SDK network-policy teardown.
expect(policyDeletes('sdk-only-workload-to-mcp-host-sdk-ingress')).toHaveLength(2)
// The first pass attempted the legacy DELETE; the second is inside the
// backoff window and sends no request.
expect(policyDeletes('sdk-only-mcp-servers-egress-internet')).toHaveLength(1)
expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(2)
expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledWith({
name: 'sdk-only-mcp-host',
namespace: sandboxNamespace,
propagationPolicy: 'Background',
})
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledTimes(4)
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'wf-sdk-only-plugin-workload-sdk-token',
namespace: sandboxNamespace,
})
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'wf-sdk-only-mcp-host-runtime-tokens',
namespace: sandboxNamespace,
})
} finally {
vi.useRealTimers()
}
})
it('R4-L3: keeps the legacy internet policy in a hybrid teardown, which preserves the workflow runtime', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await expect(
reconciler.cleanupPluginWorkloadSdk('hybrid', { preserveWorkflowRuntime: true })
).resolves.toBeUndefined()
// Liveness witness: the SDK-owned policies were deleted.
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'hybrid-workload-to-mcp-host-sdk-ingress',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
name: 'hybrid-mcp-servers-egress-internet',
namespace: mcpServerNamespace,
})
})
it('uses recipeUid as live identity and preserves the shared legacy gone-set', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
const legacyDeletes = () =>
mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
)
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('removed')
expect(legacyDeletes()).toHaveLength(1)
await reconciler.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
expect(legacyDeletes()).toHaveLength(1)
// This old-UID retry is a direct ledger probe. It does not pretend that
// Kubernetes recreated an object with the old UID.
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('removed')
expect(legacyDeletes()).toHaveLength(1)
})
it('gives finalizer cleanup one last chance and forgets only after it completes', async () => {
vi.useFakeTimers({ toFake: ['Date'] })
const reconciler = new WorkflowReconciler(makeDeps())
const legacyDeletes = () =>
mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
)
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name: string }) => {
if (name === 'sdk-only-mcp-servers-egress-internet') {
throw { code: 403, message: 'forbidden' }
}
return {}
}
)
try {
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('pending')
expect(legacyDeletes()).toHaveLength(1)
// Finalizer cleanup must attempt the DELETE even inside the live window.
const pending = await reconciler
.cleanupPluginWorkloadSdk('sdk-only', {
recipeUid: 'uid-sdk-only',
recipeDeleted: true,
})
.catch(error => error)
expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
expect(legacyDeletes()).toHaveLength(2)
// A normal old-UID probe is still inside the shared backoff. This direct
// ledger probe does not model a Kubernetes object recreation.
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('pending')
expect(legacyDeletes()).toHaveLength(2)
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => ({}))
await reconciler.cleanupPluginWorkloadSdk('sdk-only', {
recipeUid: 'uid-sdk-only',
recipeDeleted: true,
})
expect(legacyDeletes()).toHaveLength(3)
// The successful, completed finalizer cleanup forgot the old entry. The
// old-UID probe is again a direct process-local ledger witness.
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('removed')
expect(legacyDeletes()).toHaveLength(4)
} finally {
vi.useRealTimers()
}
})
it('does not forget the finalizer ledger entry when a later cleanup phase fails', async () => {
const lateError = new Error('late finalize failed')
const revocationId = 'late-finalize-revocation'
const deps = makeDeps() as unknown as {
pluginWorkloadSdkRevocationClient?: unknown
}
const revocationClient = {
revoke: vi.fn().mockResolvedValue({
state: 'revoking',
revocationId,
revoked: 1,
fencedInvocations: 1,
}),
finalize: vi.fn().mockRejectedValue(lateError),
}
deps.pluginWorkloadSdkRevocationClient = revocationClient
const reconciler = new WorkflowReconciler(deps as WorkflowReconcilerDeps)
const legacyDeletes = () =>
mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
)
await expect(
reconciler.cleanupPluginWorkloadSdk('sdk-only', {
recipeUid: 'uid-sdk-only',
recipeDeleted: true,
})
).rejects.toBe(lateError)
// Positive witnesses: the central legacy DELETE succeeded, and the later
// revocation-finalization phase—not an earlier network failure—rejected.
expect(legacyDeletes()).toHaveLength(1)
expect(revocationClient.finalize).toHaveBeenCalledWith(
sandboxNamespace,
'sdk-only',
revocationId
)
// The old-UID retry is a direct process-local ledger probe, not a
// Kubernetes recreation. It must hit the recorded gone-set, not send a
// second DELETE after only partial cleanup completion.
await expect(
reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
).resolves.toBe('removed')
expect(legacyDeletes()).toHaveLength(1)
})
it('defers absence reads until pod deletion completes so 404s cannot become unhandled rejections', async () => {
const events: string[] = []
crashRecoveryMocks.waitForPodDeletion.mockImplementationOnce(async () => {
events.push('wait-start')
await Promise.resolve()
events.push('wait-end')
return true
})
mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async () => {
events.push('network-policy-read')
throw { code: 404 }
})
const reconciler = new WorkflowReconciler(makeDeps())
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
expect(events.indexOf('wait-start')).toBeGreaterThanOrEqual(0)
expect(events.indexOf('wait-end')).toBeGreaterThan(events.indexOf('wait-start'))
expect(events.indexOf('network-policy-read')).toBeGreaterThan(events.indexOf('wait-end'))
})
it('removes only SDK-owned authority from a hybrid workflow runtime', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await expect(
reconciler.cleanupPluginWorkloadSdk('hybrid', { preserveWorkflowRuntime: true })
).resolves.toBeUndefined()
expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
name: 'wf-hybrid-plugin-workload-sdk-token',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'hybrid-workload-to-mcp-host-sdk-ingress',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'hybrid-workload-to-mcp-host-sdk-egress',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'hybrid-mcp-host-to-wrc-sdk-broker',
namespace: sandboxNamespace,
})
expect(mockCoreApi.deleteNamespacedService).not.toHaveBeenCalled()
expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
expect(mockCoreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
name: 'wf-hybrid-mcp-host-runtime-tokens',
namespace: sandboxNamespace,
})
expect(mockNetworkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
it('fails closed before physical cleanup when the Control API revocation client is absent', async () => {
const deps = makeDeps() as unknown as { pluginWorkloadSdkRevocationClient?: unknown }
delete deps.pluginWorkloadSdkRevocationClient
const reconciler = new WorkflowReconciler(deps as unknown as WorkflowReconcilerDeps)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
/revocation client is not configured/
)
expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
})
it('retains every SDK resource when Control API revocation fails', async () => {
const deps = makeDeps()
mockPluginWorkloadSdkRevocationClient.revoke.mockRejectedValueOnce(
new Error('control-api unavailable')
)
const reconciler = new WorkflowReconciler(deps)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
'control-api unavailable'
)
expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
expect(mockCoreApi.deleteNamespacedService).not.toHaveBeenCalled()
expect(mockCoreApi.deleteNamespacedSecret).not.toHaveBeenCalled()
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
it('converges after a post-revocation delete failure on the next reconcile', async () => {
const deps = makeDeps()
const revocationId = '55555555-5555-4555-8555-555555555555'
mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
state: 'revoking',
revocationId,
revoked: 1,
fencedInvocations: 1,
})
mockPluginWorkloadSdkRevocationClient.finalize.mockResolvedValue({
state: 'disabled',
revocationId,
revoked: 0,
fencedInvocations: 0,
disabled: 1,
})
mockCoreApi.deleteNamespacedService.mockRejectedValueOnce(new Error('apiserver timeout'))
const reconciler = new WorkflowReconciler(deps)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
'apiserver timeout'
)
expect(mockPluginWorkloadSdkRevocationClient.finalize).not.toHaveBeenCalled()
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledWith(
sandboxNamespace,
'sdk-only',
revocationId
)
})
it('retries finalization after a transient Control API failure once resources are absent', async () => {
const deps = makeDeps()
const revocationId = '66666666-6666-4666-8666-666666666666'
mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
state: 'revoking',
revocationId,
revoked: 1,
fencedInvocations: 0,
})
mockPluginWorkloadSdkRevocationClient.finalize
.mockRejectedValueOnce(new Error('control-api finalize timeout'))
.mockResolvedValueOnce({
state: 'disabled',
revocationId,
revoked: 0,
fencedInvocations: 0,
disabled: 1,
})
const reconciler = new WorkflowReconciler(deps)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
'control-api finalize timeout'
)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledTimes(2)
expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenLastCalledWith(
sandboxNamespace,
'sdk-only',
revocationId
)
})
it('accepts an already-removed grant as terminal revocation confirmation', async () => {
const deps = makeDeps()
const revocationId = '77777777-7777-4777-8777-777777777777'
mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
state: 'revoking',
revocationId,
revoked: 1,
fencedInvocations: 0,
})
mockPluginWorkloadSdkRevocationClient.finalize.mockResolvedValue({
state: 'missing',
revoked: 0,
fencedInvocations: 0,
})
const reconciler = new WorkflowReconciler(deps)
await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledWith(
sandboxNamespace,
'sdk-only',
revocationId
)
})
it('sweeps legacy SDK network policies by recipe label during cleanup', async () => {
let legacyPolicyPresent = true
mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(
({ namespace }: { namespace: string }) =>
Promise.resolve({
items:
namespace === sandboxNamespace && legacyPolicyPresent
? [{ metadata: { name: 'sdk-only-legacy-coord-to-mcp-host' } }]
: [],
})
)
mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
async ({ name }: { name?: string }) => {
if (name === 'sdk-only-legacy-coord-to-mcp-host') legacyPolicyPresent = false
return {}
}
)
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.cleanupPluginWorkloadSdk('sdk-only')
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: sandboxNamespace,
labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
})
expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
namespace: mcpServerNamespace,
labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
})
expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
name: 'sdk-only-legacy-coord-to-mcp-host',
namespace: sandboxNamespace,
})
})
it('brokers the provider into the eager mcp-host for promptBridge recipes', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(1)
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
'zai',
'glm-4.7',
expect.any(String),
'wrc-configure-token',
'promptBridge',
null
)
})
it('does not broker a provider for clientNotifications-only recipes but proves its grant', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: {
configured: true,
ready: true,
contractVersion: 2,
capabilityFamily: 'clientNotifications',
policyReady: true,
policyState: 'active',
},
})
await reconciler.reconcile(
'notif-recipe',
'uid-notif',
sandboxNamespace,
sdkSpec({
pluginWorkloadSdk: {
clientNotifications: { allowedEventTypes: ['e2e.test'] },
allowedCallers: ['sdk-caller'],
},
}),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'notif-recipe',
undefined
)
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('notif-recipe-mcp-host')
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
undefined,
undefined,
expect.any(String),
'wrc-configure-token',
'clientNotifications'
)
})
it('keeps clientNotifications-only recipes awaiting policy when the host proves no grant', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: {
configured: true,
ready: true,
contractVersion: 2,
capabilityFamily: 'clientNotifications',
policyReady: false,
policyState: 'missing',
policyReason: 'grant_missing',
},
})
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'notifications-policy-pending',
'uid-notifications-policy-pending',
sandboxNamespace,
sdkSpec({
agent: undefined,
steps: undefined,
pluginWorkloadSdk: {
clientNotifications: { allowedEventTypes: ['e2e.test'] },
allowedCallers: ['sdk-caller'],
},
})
)
expect(result.phase).toBe('awaiting_policy')
expect(result.message).toContain('operator policy pending')
expect(result.message).toContain('grant_missing')
})
it('skips the provider broker while the eager mcp-host is not ready', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({ ready: false, phase: 'Pending' })
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).not.toHaveBeenCalled()
})
it('returns phase=deploying while the eager mcp-host boots so the watcher requeues the configure', async () => {
// The eager pod is created but not Ready yet, so /configure is skipped this
// pass. The result MUST be phase=deploying (NOT active) so the watcher
// requeues at the fixed progress interval and re-runs the eager /configure
// once the pod is Ready. Returning 'active' drops the requeue
// (requeueAfterMs=undefined) — the now-Ready pod is never re-reconciled and
// stays "waiting for /configure" forever, so promptBridge / clientNotifications
// fail with provider_unavailable. Regression guard for the readiness race.
crashRecoveryMocks.getPodReadiness.mockResolvedValue({ ready: false, phase: 'Pending' })
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(result.phase).toBe('deploying')
expect(result.message).toContain('starting')
expect(result.message).not.toContain('registered')
})
it('returns phase=active once the eager mcp-host is Ready and configured (steady state)', async () => {
// Pod Ready + /configure succeeds → eagerStatus='ready'. The recipe is
// settled: phase=active (no progress requeue); the workflowNeedsInfrastructure
// Reconcile keep-alive loop handles steady-state re-configure/self-heal.
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(result.phase).toBe('active')
expect(result.message).toContain('registered')
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(1)
})
it('reports awaiting_policy when identity bootstrap is ready but the operator grant is missing', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: {
configured: true,
ready: true,
provider: 'zai',
model: 'glm-4.7',
contractVersion: 2,
policyReady: false,
policyState: 'missing',
policyReason: 'grant_missing',
},
})
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcilePluginWorkloadSdkOnly(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec({ steps: undefined })
)
expect(result.phase).toBe('awaiting_policy')
expect(result.message).toContain('operator policy pending')
expect(result.message).toContain('grant_missing')
expect(result.message).not.toContain('provider unavailable')
expect(result.pluginWorkloadSdkBootstrapProof).toMatchObject({
ready: true,
policyReady: false,
policyState: 'missing',
policyReason: 'grant_missing',
})
})
it('re-probes live bootstrap policy on every reconcile of the same ready pod', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
const reconciler = new WorkflowReconciler(makeDeps())
const run = () =>
reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
await run()
await run()
// A pod UID is not proof that the operator policy is unchanged. Each
// reconcile must obtain a fresh Control API proof so grant revocation and
// target-order changes become effective without restarting the pod.
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
})
it('stays pending (not registered) and retries when the provider broker fails', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 500,
body: { error: 'boom' },
})
const reconciler = new WorkflowReconciler(makeDeps())
const run = () =>
reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
const first = await run()
// Configure failed → recipe must NOT report the SDK as registered/ready.
expect(first.message).toContain('starting')
expect(first.message).not.toContain('registered')
// Failure must not poison the configure cache: the next reconcile retries.
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: sdkBootstrapProof(),
})
const second = await run()
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
expect(second.message).toContain('registered')
})
it('does not publish SDK readiness when the bootstrap contract is not v2', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: { configured: true },
})
const reconciler = new WorkflowReconciler(makeDeps())
const result = await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(result.phase).toBe('deploying')
expect(result.message).toContain('starting')
expect(result.message).not.toContain('registered')
})
it('projects provider_unavailable after repeated configure failures instead of an indefinite starting', async () => {
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
// Provider broker is permanently broken (e.g. bad secretRef).
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 500,
body: { error: 'boom' },
})
const reconciler = new WorkflowReconciler(makeDeps())
const run = () =>
reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
// First two failures stay "starting" so transient blips keep retrying.
const first = await run()
expect(first.message).toContain('starting')
expect(first.message).not.toContain('provider unavailable')
const second = await run()
expect(second.message).toContain('starting')
// Third consecutive failure crosses MAX_EAGER_CONFIGURE_ATTEMPTS → distinct
// provider_unavailable projection (still phase=active so it self-heals).
const third = await run()
expect(third.phase).toBe('active')
expect(third.message).toContain('provider unavailable')
expect(third.workflowConditions).toEqual(
expect.arrayContaining([
expect.objectContaining({
type: 'PluginWorkloadSdkProviderUnavailable',
reason: 'EagerConfigureFailed',
status: 'True',
}),
])
)
// It keeps retrying — recovers as soon as the broker config is fixed, and
// the unavailable condition is no longer projected.
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 202,
body: sdkBootstrapProof(),
})
const recovered = await run()
expect(recovered.message).toContain('registered')
expect(recovered.workflowConditions ?? []).not.toEqual(
expect.arrayContaining([
expect.objectContaining({ type: 'PluginWorkloadSdkProviderUnavailable' }),
])
)
})
it('restarts the configure failure budget when the eager mcp-host pod is externally replaced', async () => {
let podUid = 'pod-uid-1'
crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
if (podName.endsWith('-mcp-host')) {
return { ready: true, phase: 'Running', uid: podUid }
}
return { ready: true, phase: 'Running' }
})
mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
status: 500,
body: { error: 'boom' },
})
const reconciler = new WorkflowReconciler(makeDeps())
const run = () =>
reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect((await run()).message).toContain('starting')
expect((await run()).message).toContain('starting')
// The replacement was not rolled by WRC, so no explicit reset hook runs.
// The budget must still restart because it is keyed by pod UID/provider/model.
podUid = 'pod-uid-2'
const replacementFirstFailure = await run()
expect(replacementFirstFailure.message).toContain('starting')
expect(replacementFirstFailure.message).not.toContain('provider unavailable')
expect((await run()).message).toContain('starting')
const replacementThirdFailure = await run()
expect(replacementThirdFailure.message).toContain('provider unavailable')
})
it('re-brokers the provider after the eager mcp-host pod restarts (new uid)', async () => {
const reconciler = new WorkflowReconciler(makeDeps())
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-1',
})
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
// Pod restarted — new UID invalidates the cache, forcing a fresh configure.
crashRecoveryMocks.getPodReadiness.mockResolvedValue({
ready: true,
phase: 'Running',
uid: 'pod-uid-2',
})
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
})
it('recreates a wedged (CrashLoopBackOff) eager mcp-host pod', async () => {
// Pod exists and is Running but stuck in CrashLoopBackOff — the terminal-phase
// guard alone would never replace it, so the eager path must delete it first.
crashRecoveryMocks.getPodPhase.mockResolvedValue('Running')
crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue('CrashLoopBackOff')
const reconciler = new WorkflowReconciler(makeDeps())
await reconciler.reconcile(
'sdk-recipe',
'uid-sdk',
sandboxNamespace,
sdkSpec(),
{ workflowExecution: { phase: 'initializing' } },
undefined,
'sdk-recipe',
undefined
)
expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
mockCoreApi,
'sdk-recipe-mcp-host',
sandboxNamespace
)
// After deletion the pod is recreated in the same pass.
const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
call => call[0].body.metadata.name
)
expect(podNames).toContain('sdk-recipe-mcp-host')
})
})