Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
119 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { buildMcpHostHeadlessService } from './podFactory'
import { buildMcpHostServiceName } from './resourceNames'
import {
  LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,
  LegacyNetworkPolicyDeletePendingError,
  WorkflowReconciler,
  type WorkflowReconcilerDeps,
} from './workflowReconciler'

const crashRecoveryMocks = vi.hoisted(() => ({
  deletePodIfExists: vi.fn().mockResolvedValue(undefined),
  waitForPodDeletion: vi.fn().mockResolvedValue(true),
  evaluateCompletedRuntimePodRecovery: vi.fn().mockReturnValue({
    action: 'replace',
    message:
      'mcp_host pod completed before workflow became terminal; creating replacement (attempt 1/3)',
    newPhase: 'recovering',
    newAttempt: 1,
  }),
  evaluateCrashRecovery: vi.fn().mockReturnValue({ action: 'none', message: 'Pod is healthy' }),
  getContainerWaitingReason: vi.fn().mockResolvedValue(undefined),
  getPodPhase: vi.fn().mockResolvedValue(undefined),
  getPodPresence: vi.fn(async (api: unknown, name: string, namespace: string) => {
    const phase = await crashRecoveryMocks.getPodPhase(api, name, namespace)
    return phase === undefined ? { kind: 'absent' as const } : { kind: 'present' as const, phase }
  }),
  getPodReadiness: vi.fn().mockResolvedValue({ ready: true, phase: 'Running', uid: 'pod-uid-1' }),
  isRecoverableContainerWaitingReason: vi.fn(
    (phase: string | undefined, reason: string | undefined) =>
      Boolean(
        reason &&
        [
          'CreateContainerConfigError',
          'CreateContainerError',
          'ImagePullBackOff',
          'ErrImagePull',
          'InvalidImageName',
          'CrashLoopBackOff',
        ].includes(reason) &&
        (phase === 'Pending' || (phase === 'Running' && reason === 'CrashLoopBackOff'))
      )
  ),
}))

const runtimeTokenIssuerMocks = vi.hoisted(() => ({
  issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
    accessToken: 'runtime-access-token',
    refreshToken: 'runtime-refresh-token',
    mcpHostControlToken: 'mcp-host-control-token',
  }),
  issueMcpHostWorkflowControlToken: vi.fn().mockResolvedValue('mcp-host-control-token'),
}))

vi.mock('./crashRecovery', () => crashRecoveryMocks)
vi.mock('./mcpHostRuntimeTokenIssuerClient', () => runtimeTokenIssuerMocks)

const gfsBindingMocks = vi.hoisted(() => ({
  mintRecipeHostGfsToken: vi.fn().mockResolvedValue({
    ['to'.concat('ken')]: 'gfs-runtime-value',
    expiresInSeconds: 300,
    subject: 'host:3rd:sandbox-recipes/workflow-recipe',
  }),
}))

vi.mock('../gfsBinding', () => gfsBindingMocks)

const isoDateAgo = (ageMs: number) => new Date(Date.now() - ageMs).toISOString()

function unsignedRuntimeJwt(payload: Record<string, unknown>): string {
  return `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify(payload)).toString('base64url')}.x`
}

type MockPodReadiness = {
  ready: boolean
  phase?: string
  uid?: string
  waitingReason?: string
  schedulingReason?: string
}

function mockAgenticOutputPodPhases(overrides?: {
  anchor?: string
  prepare?: string
  mcpHost?: string | (() => string | undefined)
}) {
  crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
    if (name === 'agentic-recipe-workflow-output-anchor') {
      return overrides?.anchor ?? 'Running'
    }
    if (name === 'agentic-recipe-workflow-output-prepare') {
      return overrides?.prepare ?? 'Succeeded'
    }
    if (name === 'agentic-recipe-mcp-host') {
      const value = overrides?.mcpHost
      return typeof value === 'function' ? value() : (value ?? undefined)
    }
    return undefined
  })
}

function mockMcpHostReadiness(readiness: MockPodReadiness) {
  crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
    if (podName.endsWith('-mcp-host')) {
      return readiness
    }
    return { ready: true, phase: 'Running' }
  })
}

function mockMcpHostReadinessSequence(...sequence: MockPodReadiness[]) {
  let mcpHostCalls = 0
  crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
    if (podName.endsWith('-mcp-host')) {
      const value = sequence[Math.min(mcpHostCalls, sequence.length - 1)]
      mcpHostCalls += 1
      return value
    }
    return { ready: true, phase: 'Running' }
  })
}

describe('WorkflowReconciler.reconcileDelete — orphaned Service cleanup', () => {
  const sandboxNamespace = 'sandbox-recipes'
  const mcpServerNamespace = 'mcp-server'

  const mockCoreApi = {
    readNamespacedPod: vi.fn().mockResolvedValue({}),
    readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
    readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
    readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
    createNamespacedSecret: vi.fn().mockResolvedValue({}),
    createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
    createNamespacedService: vi.fn().mockResolvedValue({}),
    createNamespacedPod: vi.fn().mockResolvedValue({}),
    patchNamespacedSecret: vi.fn().mockResolvedValue({}),
    deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
    deleteNamespacedConfigMap: vi.fn().mockResolvedValue({}),
    deleteNamespacedService: vi.fn().mockResolvedValue({}),
    readNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({
      metadata: { name: 'existing-workflow-output', deletionTimestamp: undefined },
    }),
    createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
    deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
    // Under test: new label-selector cleanup for cross-namespace-orphaned Services.
    deleteCollectionNamespacedService: vi.fn().mockResolvedValue({}),
  }

  const mockNetworkingApi = {
    createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
    replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
  }

  const mockCustomApi = {
    patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
    patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
  }

  // Minimal token factory for the mcp-host artifact cleanup path.
  const mockTokenFactory = {
    signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('stub-token'),
    signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('coordinator-mcp-host-token'),
    signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('custom-coordinator-wrc-token'),
    signCoordinatorToWrcToken: vi.fn().mockResolvedValue('coordinator-wrc-token'),
  }

  const deps = {
    coreApi: mockCoreApi,
    customApi: mockCustomApi,
    networkingApi: mockNetworkingApi,
    config: {
      coordinatorImage: 'coordinator:test',
      mcpHostImage: 'mcp-host:test',
      wrcEndpoint: 'http://wrc.example/api',
      sandboxNamespace,
      mcpServerNamespace,
      imagePullPolicy: 'IfNotPresent' as const,
      maxWorkflowSteps: 100,
      runtimeTokenTtlSeconds: 3600,
      runtimeTokenRefreshBeforeSeconds: 300,
    },
    tokenFactory: mockTokenFactory,
    pluginWorkloadSdkRevocationClient: {
      revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),
      finalize: vi.fn(),
    },
  } as unknown as WorkflowReconcilerDeps

  beforeEach(() => {
    vi.clearAllMocks()
    crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    // Run-scoped reconciles skip the eager→run recreate gate when the mcp-host already
    // carries the workflow-output claim label.
    mockCoreApi.readNamespacedPod.mockImplementation(async (params: { name?: string }) => {
      if (typeof params.name === 'string' && params.name.endsWith('-mcp-host')) {
        return {
          metadata: {
            labels: { 'clerum.io/workflow-output-claim': 'shared-output' },
          },
        }
      }
      return {}
    })
    mockCoreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
    // No Service exists until a test says so: reconcile reads before it
    // creates, so a resolved read would stand for a live (spec-less) Service.
    mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
    mockCoreApi.createNamespacedSecret.mockResolvedValue({})
    mockCoreApi.createNamespacedConfigMap.mockResolvedValue({})
    mockCoreApi.createNamespacedService.mockResolvedValue({})
    mockCoreApi.createNamespacedPod.mockResolvedValue({})
    mockCoreApi.patchNamespacedSecret.mockResolvedValue({})
    mockNetworkingApi.createNamespacedNetworkPolicy.mockResolvedValue({})
    mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: { resourceVersion: '1' },
    })
    mockNetworkingApi.replaceNamespacedNetworkPolicy.mockResolvedValue({})
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})
    mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
    mockCustomApi.patchNamespacedCustomObject.mockResolvedValue({})
    mockCustomApi.patchNamespacedCustomObjectStatus.mockResolvedValue({})
    crashRecoveryMocks.evaluateCrashRecovery.mockReturnValue({
      action: 'none',
      message: 'Pod is healthy',
    })
    crashRecoveryMocks.evaluateCompletedRuntimePodRecovery.mockReturnValue({
      action: 'replace',
      message:
        'mcp_host pod completed before workflow became terminal; creating replacement (attempt 1/3)',
      newPhase: 'recovering',
      newAttempt: 1,
    })
    crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    crashRecoveryMocks.isRecoverableContainerWaitingReason.mockImplementation(
      (phase: string | undefined, reason: string | undefined) =>
        Boolean(
          reason &&
          [
            'CreateContainerConfigError',
            'CreateContainerError',
            'ImagePullBackOff',
            'ErrImagePull',
            'InvalidImageName',
            'CrashLoopBackOff',
          ].includes(reason) &&
          (phase === 'Pending' || (phase === 'Running' && reason === 'CrashLoopBackOff'))
        )
    )
    runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockResolvedValue({
      accessToken: 'runtime-access-token',
      refreshToken: 'runtime-refresh-token',
      mcpHostControlToken: 'mcp-host-control-token',
    })
    runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken.mockResolvedValue(
      'mcp-host-control-token'
    )
    // Stub global.fetch used by cleanupRecipeArtifacts (best-effort).
    vi.stubGlobal(
      'fetch',
      vi.fn().mockResolvedValue({ ok: true, status: 204 } as unknown as Response)
    )
  })

  it('stores inherited child mcp-host control credentials with the runtime-scope caller', async () => {
    const reconciler = new WorkflowReconciler(deps)
    runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens.mockResolvedValueOnce({
      accessToken: 'parent-runtime-access-token',
      refreshToken: 'parent-runtime-refresh-token',
      mcpHostControlToken: 'parent-workflow-control-token',
    })
    runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken.mockResolvedValueOnce(
      'child-workflow-control-token'
    )

    await reconciler.ensureMcpHostRuntimeCredentials(
      sandboxNamespace,
      'child-run',
      { steps: [{ id: 'step-1', instruction: 'run with tools' }] },
      'parent-recipe'
    )

    expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
      sandboxNamespace,
      'parent-recipe',
      []
    )
    expect(runtimeTokenIssuerMocks.issueMcpHostWorkflowControlToken).not.toHaveBeenCalled()
    const createCall = mockCoreApi.createNamespacedSecret.mock.calls[0]?.[0] as {
      body?: { data?: Record<string, string> }
    }
    expect(
      Buffer.from(
        createCall.body?.data?.['mcp-host-workflow-control-token'] ?? '',
        'base64'
      ).toString('utf-8')
    ).toBe('parent-workflow-control-token')
  })

  it('mints read-scoped GFS credentials for recipe hosts by default', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-recipe', {
      steps: [{ id: 'step-1', instruction: 'read inputs' }],
    })

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'read-recipe',
      { scopes: ['gfs.read'] }
    )
  })

  it('mints exactly read scope when a recipe mount is read-only', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-mount-recipe', {
      steps: [{ id: 'step-1', instruction: 'read mounted inputs' }],
      gfs: {
        mounts: [{ drive: 'main', target: 'inputs', scopes: ['gfs.read'] }],
      },
    })

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'read-mount-recipe',
      { scopes: ['gfs.read'] }
    )
  })

  it('keeps the read ceiling and adds write when a recipe mount is write-only', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'write-mount-recipe', {
      steps: [{ id: 'step-1', instruction: 'write mounted outputs' }],
      gfs: {
        mounts: [{ drive: 'main', target: 'outputs', scopes: ['gfs.write'] }],
      },
    })

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'write-mount-recipe',
      { scopes: ['gfs.read', 'gfs.write'] }
    )
  })

  it('orders and de-duplicates read-write mount scopes before minting', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'read-write-mount-recipe', {
      steps: [{ id: 'step-1', instruction: 'read and write mounted content' }],
      gfs: {
        mounts: [
          {
            drive: 'main',
            target: 'workspace',
            scopes: ['gfs.write', 'gfs.read', 'gfs.write'],
          },
        ],
      },
    })

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'read-write-mount-recipe',
      { scopes: ['gfs.read', 'gfs.write'] }
    )
  })

  it('mints inherited child GFS scopes against the parent runtime identity', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(
      sandboxNamespace,
      'child-run',
      {
        steps: [{ id: 'step-1', instruction: 'use the parent grant' }],
        gfs: {
          mounts: [
            {
              drive: 'main',
              target: 'workspace',
              scopes: ['gfs.read', 'gfs.write'],
            },
          ],
        },
      },
      'parent-recipe'
    )

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledTimes(1)
    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'parent-recipe',
      { scopes: ['gfs.read', 'gfs.write'] }
    )
  })

  it('mints write-scoped GFS credentials when the recipe declares publish targets', async () => {
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'publish-recipe', {
      steps: [{ id: 'step-1', instruction: 'publish output' }],
      gfs: {
        publishTargets: [{ drive: 'main', target: 'outputs' }],
      },
    })

    expect(gfsBindingMocks.mintRecipeHostGfsToken).toHaveBeenCalledWith(
      sandboxNamespace,
      'publish-recipe',
      { scopes: ['gfs.read', 'gfs.write'] }
    )
  })

  it('deletes transport Services in mcp-server by label selector (agentic path, cross-namespace orphans)', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const recipeName = 'my-agentic-recipe'

    await reconciler.reconcileDelete(recipeName, sandboxNamespace)

    // The new cleanup call: mcp-server namespace + composite label selector
    // matching what `buildService()` actually emits on Services (NOT `wrc`,
    // which is only on NetworkPolicies built by workflowReconciler itself).
    expect(mockCoreApi.deleteCollectionNamespacedService).toHaveBeenCalledTimes(1)
    expect(mockCoreApi.deleteCollectionNamespacedService).toHaveBeenCalledWith({
      namespace: mcpServerNamespace,
      labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=workflow-recipes`,
    })
  })

  it('still deletes per-mcp-server NetworkPolicies by label selector without deletecollection', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const recipeName = 'my-agentic-recipe'
    mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(({ namespace }) =>
      Promise.resolve({
        items:
          namespace === mcpServerNamespace
            ? [{ metadata: { name: `${recipeName}-dynamic-mcp-ingress` } }]
            : [],
      })
    )

    await reconciler.reconcileDelete(recipeName, sandboxNamespace)

    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: mcpServerNamespace,
      labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: `${recipeName}-dynamic-mcp-ingress`,
      namespace: mcpServerNamespace,
    })
  })

  it('deletes WRC-managed sandbox NetworkPolicies by recipe label for dynamic snippet policy names', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const recipeName = 'my-snippet-recipe'
    mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(({ namespace }) =>
      Promise.resolve({
        items:
          namespace === sandboxNamespace
            ? [{ metadata: { name: `${recipeName}-dynamic-snippet-ingress` } }]
            : [],
      })
    )

    await reconciler.reconcileDelete(recipeName, sandboxNamespace)

    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: sandboxNamespace,
      labelSelector: `clerum.io/recipe=${recipeName},clerum.io/managed-by=wrc`,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: `${recipeName}-dynamic-snippet-ingress`,
      namespace: sandboxNamespace,
    })
  })

  it('prefers artifact-reader HTTP artifact cleanup when the artifact-reader Service exists', async () => {
    mockCoreApi.readNamespacedService.mockResolvedValue({})
    const reconciler = new WorkflowReconciler(deps)
    const recipeName = 'agentic-recipe'

    await reconciler.reconcileDelete(recipeName, sandboxNamespace)

    expect(mockTokenFactory.signWrcArtifactDeleteToken).toHaveBeenCalledWith(
      recipeName,
      sandboxNamespace
    )
    expect(fetch).toHaveBeenCalledWith(
      `http://wf-${recipeName}-artifact-reader.${sandboxNamespace}.svc.cluster.local:8080/api/v1/workflow/artifacts`,
      expect.objectContaining({
        method: 'DELETE',
        headers: { authorization: 'Bearer stub-token' },
      })
    )
  })

  it('accepts snippet secretRef without requiring a matching spec.resources secret placeholder', () => {
    const reconciler = new WorkflowReconciler(deps)

    const error = reconciler.validateWorkflowSpec({
      steps: [
        {
          id: 'fetch-price',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'coingecko_api_key',
                  secretRef: { name: 'coingecko-api', key: 'apiKey' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(error).toBeUndefined()
  })

  it('rejects snippet secretRef entries that point at platform-managed workflow Secrets', () => {
    const reconciler = new WorkflowReconciler(deps)

    const error = reconciler.validateWorkflowSpec({
      steps: [
        {
          id: 'read-runtime-token',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'runtime_token',
                  secretRef: { name: 'wf-example-coordinator-token', key: 'token' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(error).toBe(
      'step "read-runtime-token" cannot reference platform-managed secret "wf-example-coordinator-token"'
    )
  })

  it('does not reject user-managed snippet Secrets only because their name mentions coordinator-token', () => {
    const reconciler = new WorkflowReconciler(deps)

    const error = reconciler.validateWorkflowSpec({
      steps: [
        {
          id: 'read-api-key',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'api_key',
                  secretRef: { name: 'api-coordinator-token-rotator', key: 'token' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(error).toBeUndefined()
  })

  it('fails workflow reconcile before pod creation when a snippet secretRef Secret is missing', async () => {
    mockCoreApi.readNamespacedSecret.mockRejectedValueOnce({ code: 404 })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile('snippet-recipe', 'uid-snippet', sandboxNamespace, {
      steps: [
        {
          id: 'fetch-price',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'coingecko_api_key',
                  secretRef: { name: 'coingecko-api', key: 'apiKey' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
      message: 'snippet secret "coingecko-api" was not found in namespace "sandbox-recipes"',
    })
    expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
      name: 'coingecko-api',
      namespace: sandboxNamespace,
    })
  })

  it('resolves snippet resource ids through status.resourceInstances before validating access', async () => {
    mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
      metadata: { labels: { 'clerum.io/owner-recipe': 'snippet-recipe' } },
      data: Object.fromEntries([['pass', 'present']]),
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'snippet-recipe',
      'uid-snippet',
      sandboxNamespace,
      {
        resources: [{ id: 'pg-auth', type: 'secret', data: { pass: 'redacted' } }],
        steps: [
          {
            id: 'query-postgres',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return { ok: true }',
              capabilities: {
                secrets: [{ alias: 'pg', secretRef: { name: 'pg-auth', key: 'pass' } }],
              },
            },
          },
        ],
      },
      { resourceInstances: { 'pg-auth': 'snippet-recipe-pg-auth-d07f008b65ec' } }
    )

    expect(result.phase).toBe('deploying')
    expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
      name: 'snippet-recipe-pg-auth-d07f008b65ec',
      namespace: sandboxNamespace,
    })
  })

  it('preserves workflow status and signals skipStatusPatch on a transient infra error (no terminal failure)', async () => {
    const reconciler = new WorkflowReconciler(deps)
    // First call inside the reconcile try (getPodPhase) hits a transient API blip.
    crashRecoveryMocks.getPodPhase.mockRejectedValueOnce(
      Object.assign(new Error('connect ETIMEDOUT 10.96.0.1:443'), { code: 'ETIMEDOUT' })
    )

    const result = await reconciler.reconcile(
      'flaky-workflow',
      'uid-flaky',
      sandboxNamespace,
      {
        steps: [
          { id: 'render', run: { type: 'snippet', language: 'typescript', code: 'return {}' } },
        ],
      },
      { workflowExecution: { phase: 'recovering', message: 'prior transient blip' } }
    )

    // Must NOT latch the workflow to terminal failed.
    expect(result.phase).not.toBe('failed')
    expect(result.workflowPhase).not.toBe('failed')
    // Preserves the current execution phase + message and tells WRC to skip the patch.
    expect(result.skipStatusPatch).toBe(true)
    expect(result.workflowPhase).toBe('recovering')
    expect(result.message).toBe('prior transient blip')
    // No status patch was written for the transient blip.
    expect(mockCustomApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
  })

  it('fails workflow reconcile before pod creation when a snippet secretRef key is missing', async () => {
    // Owned by the recipe (passes the Issue #637 ownership gate) but the key is
    // absent — so the failure must be the key-missing message, not ownership.
    mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
      metadata: { labels: { 'clerum.io/owner-recipe': 'snippet-recipe' } },
      data: { otherKey: 'dmFsdWU=' },
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile('snippet-recipe', 'uid-snippet', sandboxNamespace, {
      steps: [
        {
          id: 'fetch-price',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'coingecko_api_key',
                  secretRef: { name: 'coingecko-api', key: 'apiKey' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
      message:
        'snippet secret key "apiKey" was not found in Secret "coingecko-api" in namespace "sandbox-recipes"',
    })
  })

  it('fails workflow reconcile before pod creation when a snippet secretRef Secret is owned by another recipe (Issue #637)', async () => {
    // The Secret EXISTS and HAS the requested key, but it is owned by a DIFFERENT
    // recipe — the snippet runner pod must never be created with a foreign credential.
    mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
      metadata: { labels: { 'clerum.io/owner-recipe': 'victim-recipe' } },
      data: { apiKey: 'c3RvbGVu' },
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile('attacker-recipe', 'uid-attacker', sandboxNamespace, {
      steps: [
        {
          id: 'fetch-price',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return { ok: true }',
            capabilities: {
              secrets: [
                {
                  alias: 'coingecko_api_key',
                  secretRef: { name: 'coingecko-api', key: 'apiKey' },
                },
              ],
            },
          },
        },
      ],
    })

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
      message:
        'snippet secret "coingecko-api" is not accessible to recipe "attacker-recipe" — ' +
        'it requires clerum.io/shared=true or clerum.io/owner-recipe=attacker-recipe',
    })
    // No snippet-runner pod was created for the denied recipe.
    expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  describe('snippet secret ownership on a triggered run (run-scoped child CR)', () => {
    // A triggered run reconciles the run-scoped child CR `<parent>-<runid8>`, while
    // WRC resolves the verified parent recipe as the runtime scope. Snippet Secret
    // ownership must be judged against that parent scope, never the child name.
    const childRecipeName = 'pr-review-abcd1234'
    const parentScopeRecipeName = 'pr-review'
    const triggeredRunId = 'run-id'
    const scopeDenialMessage =
      'snippet secret "github-token" is not accessible to recipe "pr-review" — ' +
      'it requires clerum.io/shared=true or clerum.io/owner-recipe=pr-review'

    function snippetSecretSpec(secretName = 'github-token') {
      return {
        steps: [
          {
            id: 'triage',
            run: {
              type: 'snippet' as const,
              language: 'typescript' as const,
              code: 'return { ok: true }',
              capabilities: {
                secrets: [{ alias: 'github_token', secretRef: { name: secretName, key: 'token' } }],
              },
            },
          },
        ],
      }
    }

    function reconcileTriggeredChild(
      spec = snippetSecretSpec(),
      currentStatus?: { resourceInstances?: Record<string, string> }
    ) {
      // Settle the output gates so a pass that clears the preflight reaches runtime
      // pod creation. The two names differ in scope ON PURPOSE: the output claim and
      // its anchor pod belong to the PARENT scope (`pr-review-workflow-output-anchor`),
      // while the prepare pod is RUN-scoped (`pr-review-abcd1234-workflow-output-prepare`).
      // Without these, the pass stops at "Created workflow output anchor pod" and
      // the snippet-runner pod is never created.
      crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
        if (name === `${parentScopeRecipeName}-workflow-output-anchor`) return 'Running'
        if (name === `${childRecipeName}-workflow-output-prepare`) return 'Succeeded'
        return undefined
      })
      return new WorkflowReconciler(deps).reconcile(
        childRecipeName,
        'uid-pr-review-child',
        sandboxNamespace,
        spec,
        currentStatus,
        undefined,
        parentScopeRecipeName,
        triggeredRunId
      )
    }

    function createdPodNames(): string[] {
      return mockCoreApi.createNamespacedPod.mock.calls.map(call => call[0].body.metadata.name)
    }

    it('creates the snippet runner when the Secret is owned by the parent scope recipe', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: { 'clerum.io/owner-recipe': parentScopeRecipeName } },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result.phase).not.toBe('failed')
      expect(result.workflowPhase).not.toBe('failed')
      expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
        name: 'github-token',
        namespace: sandboxNamespace,
      })
      expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
      const runnerPod = mockCoreApi.createNamespacedPod.mock.calls
        .map(call => call[0].body)
        .find(body => body.metadata.name === `${childRecipeName}-snippet-runner`)
      expect(runnerPod).toBeDefined()
      const projectedSecretRefs = runnerPod.spec.containers.flatMap(
        (container: { env?: Array<{ valueFrom?: { secretKeyRef?: unknown } }> }) =>
          (container.env ?? []).flatMap(env =>
            env.valueFrom?.secretKeyRef ? [env.valueFrom.secretKeyRef] : []
          )
      )
      expect(projectedSecretRefs).toEqual([{ name: 'github-token', key: 'token' }])
    })

    it('fails before pod creation when the Secret is owned by another recipe, naming the parent scope', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: { 'clerum.io/owner-recipe': 'other-recipe' } },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result).toMatchObject({
        phase: 'failed',
        workflowPhase: 'failed',
        message: scopeDenialMessage,
      })
      expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    })

    it('fails before pod creation when the Secret is owned by the run-scoped child name', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: { 'clerum.io/owner-recipe': childRecipeName } },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result).toMatchObject({
        phase: 'failed',
        workflowPhase: 'failed',
        message: scopeDenialMessage,
      })
      expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    })

    it('fails closed when the Secret carries both owner-recipe=<parent> and shared=true', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: {
          labels: {
            'clerum.io/owner-recipe': parentScopeRecipeName,
            'clerum.io/shared': 'true',
          },
        },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result).toMatchObject({
        phase: 'failed',
        workflowPhase: 'failed',
        message: scopeDenialMessage,
      })
      expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    })

    it('fails closed when the Secret carries no ownership label', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: {} },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result).toMatchObject({
        phase: 'failed',
        workflowPhase: 'failed',
        message: scopeDenialMessage,
      })
      expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    })

    it('keeps shared=true Secrets accessible to the triggered run', async () => {
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: { 'clerum.io/shared': 'true' } },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild()

      expect(result.phase).not.toBe('failed')
      expect(result.workflowPhase).not.toBe('failed')
      expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
    })

    it('judges a Secret resolved through a supplied resourceInstances map against the parent scope', async () => {
      // Pins only the preflight: given a resourceInstances map, the snippet resource id
      // resolves through it and ownership is judged against the parent scope. The map
      // is supplied by hand; how a triggered child builds it is not covered here.
      mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
        metadata: { labels: { 'clerum.io/owner-recipe': parentScopeRecipeName } },
        data: { token: 'fixture-token' },
      })

      const result = await reconcileTriggeredChild(
        {
          ...snippetSecretSpec('gh-auth'),
          resources: [{ id: 'gh-auth', type: 'secret' as const, data: { token: 'fixture-token' } }],
        },
        { resourceInstances: { 'gh-auth': 'pr-review-gh-auth-d07f008b65ec' } }
      )

      expect(result.phase).not.toBe('failed')
      expect(mockCoreApi.readNamespacedSecret).toHaveBeenCalledWith({
        name: 'pr-review-gh-auth-d07f008b65ec',
        namespace: sandboxNamespace,
      })
      expect(createdPodNames()).toContain(`${childRecipeName}-snippet-runner`)
    })

    it('still fails closed (throws) when Secret ownership cannot be read', async () => {
      mockCoreApi.readNamespacedSecret.mockRejectedValueOnce({ code: 503 })

      await expect(reconcileTriggeredChild()).rejects.toThrow(
        'snippet secret "github-token" ownership could not be verified in namespace "sandbox-recipes"'
      )
      expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    })
  })

  it('skips artifact HTTP cleanup when neither artifact-reader nor mcp-host Service exists', async () => {
    mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
    const reconciler = new WorkflowReconciler(deps)
    const recipeName = 'pure-custom-recipe'

    await reconciler.reconcileDelete(recipeName, sandboxNamespace)

    expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
      name: `wf-${recipeName}-artifact-reader`,
      namespace: sandboxNamespace,
    })
    expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
      name: `wf-${recipeName}-mcp-host`,
      namespace: sandboxNamespace,
    })
    expect(mockTokenFactory.signWrcArtifactDeleteToken).not.toHaveBeenCalled()
    expect(fetch).not.toHaveBeenCalled()
    expect(mockCoreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      name: `${recipeName}-workflow-output`,
      namespace: sandboxNamespace,
    })
  })

  it('gates coordinator creation while the mcp-host pod is still waiting for scheduling', async () => {
    crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
      if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
      if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
      return undefined
    })
    crashRecoveryMocks.getPodReadiness.mockResolvedValueOnce({
      ready: false,
      phase: 'Pending',
      schedulingReason: 'Unschedulable',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'deploying',
      workflowPhase: 'initializing',
    })
    expect(result.message).toContain('Waiting for mcp-host pod to become Ready')
    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('agentic-recipe-mcp-host')
    expect(podNames).not.toContain('agentic-recipe-coordinator')
  })

  it('preserves required toolChoice in the workflow config ConfigMap', async () => {
    crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
      if (name === 'tool-choice-recipe-workflow-output-anchor') return 'Running'
      if (name === 'tool-choice-recipe-workflow-output-prepare') return 'Succeeded'
      return undefined
    })
    mockMcpHostReadiness({ ready: false, phase: 'Pending' })
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.reconcile(
      'tool-choice-recipe',
      'uid-tool-choice',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [
          {
            id: 'use-gfs',
            instruction: 'Use the available GFS tools',
            toolChoice: 'required',
          },
        ],
      },
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'tool-choice-recipe',
      'run-1'
    )

    const configMap = mockCoreApi.createNamespacedConfigMap.mock.calls
      .map(call => call[0].body)
      .find(body => body.metadata?.name === 'tool-choice-recipe-workflow-config')
    const workflowConfig = JSON.parse(configMap.data['config.json'])

    expect(workflowConfig.steps[0]).toMatchObject({
      id: 'use-gfs',
      toolChoice: 'required',
    })
  })

  it('does not add toolChoice to workflow config steps when it is absent', async () => {
    crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
      if (name === 'default-tool-choice-recipe-workflow-output-anchor') return 'Running'
      if (name === 'default-tool-choice-recipe-workflow-output-prepare') return 'Succeeded'
      return undefined
    })
    mockMcpHostReadiness({ ready: false, phase: 'Pending' })
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.reconcile(
      'default-tool-choice-recipe',
      'uid-default-tool-choice',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'answer', instruction: 'Answer without a tool requirement' }],
      },
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'default-tool-choice-recipe',
      'run-1'
    )

    const configMap = mockCoreApi.createNamespacedConfigMap.mock.calls
      .map(call => call[0].body)
      .find(body => body.metadata?.name === 'default-tool-choice-recipe-workflow-config')
    const workflowConfig = JSON.parse(configMap.data['config.json'])

    expect(workflowConfig.steps[0]).not.toHaveProperty('toolChoice')
  })

  it('preserves recovering while the mcp-host pod is not ready', async () => {
    crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
      if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
      if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
      return undefined
    })
    crashRecoveryMocks.getPodReadiness.mockResolvedValueOnce({
      ready: false,
      phase: 'Pending',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'recovering', attempt: 1 } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'deploying',
      workflowPhase: 'recovering',
    })
  })

  it('fails stale pre-coordinator readiness wait for a Running non-ready mcp-host pod', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Running' })
    mockMcpHostReadiness({
      ready: false,
      phase: 'Running',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(10 * 60_000) } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
    })
    expect(result.message).toContain('Waiting for mcp-host pod to become Ready (phase=Running)')
    expect(result.message).toContain('readiness deadline exceeded')
  })

  it('fails stale pre-coordinator readiness wait for a Pending non-recoverable mcp-host pod', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Pending' })
    mockMcpHostReadiness({
      ready: false,
      phase: 'Pending',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(10 * 60_000) } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
    })
    expect(result.message).toContain('Waiting for mcp-host pod to become Ready (phase=Pending)')
    expect(result.message).toContain('readiness deadline exceeded')
  })

  it('keeps waiting when mcp-host readiness wait is still inside the deadline', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Running' })
    mockMcpHostReadiness({
      ready: false,
      phase: 'Running',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing', startedAt: isoDateAgo(60_000) } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'deploying',
      workflowPhase: 'initializing',
    })
    expect(result.message).not.toContain('readiness deadline exceeded')
  })

  it('does not fail the first mcp-host readiness wait before startedAt is persisted', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Pending' })
    mockMcpHostReadiness({
      ready: false,
      phase: 'Pending',
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'deploying',
      workflowPhase: 'initializing',
    })
    expect(result.message).not.toContain('readiness deadline exceeded')
  })

  it('does not gate an already running workflow on transient mcp-host readiness', async () => {
    crashRecoveryMocks.getPodPhase
      .mockResolvedValueOnce('Running')
      .mockResolvedValueOnce('Running')
      .mockResolvedValueOnce('Succeeded')
      .mockResolvedValueOnce(undefined)
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'running', attempt: 0 } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(crashRecoveryMocks.getPodReadiness).not.toHaveBeenCalled()
    expect(result).toMatchObject({
      phase: 'active',
      workflowPhase: 'running',
    })
  })

  it('does not replace a completed mcp-host after workflow completion', async () => {
    crashRecoveryMocks.getPodPhase.mockImplementation(async (_api, name: string) => {
      if (name === 'agentic-recipe-coordinator') return 'Succeeded'
      if (name === 'agentic-recipe-workflow-output-anchor') return 'Running'
      if (name === 'agentic-recipe-workflow-output-prepare') return 'Succeeded'
      if (name === 'agentic-recipe-mcp-host') return 'Succeeded'
      return undefined
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'completed', attempt: 0 } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(crashRecoveryMocks.evaluateCompletedRuntimePodRecovery).not.toHaveBeenCalled()
    expect(crashRecoveryMocks.getPodReadiness).not.toHaveBeenCalled()
    expect(crashRecoveryMocks.deletePodIfExists).not.toHaveBeenCalled()
    expect(result).toMatchObject({
      phase: 'active',
      workflowPhase: 'completed',
    })
  })

  it('creates the coordinator on a later reconcile after mcp-host becomes Ready', async () => {
    let mcpHostPhaseReads = 0
    mockAgenticOutputPodPhases({
      mcpHost: () => {
        mcpHostPhaseReads += 1
        return mcpHostPhaseReads === 1 ? undefined : 'Running'
      },
    })
    mockMcpHostReadinessSequence(
      { ready: false, phase: 'Pending' },
      { ready: true, phase: 'Running' }
    )
    const reconciler = new WorkflowReconciler(deps)
    const spec = {
      agent: { provider: 'openai' as const, model: 'gpt-4.1' },
      steps: [{ id: 'brief', instruction: 'write the brief' }],
    }

    await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      spec,
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'agentic-recipe',
      'run-1'
    )
    mockCoreApi.createNamespacedPod.mockClear()

    const result = await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      spec,
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(result).toMatchObject({
      phase: 'deploying',
      workflowPhase: 'initializing',
    })
    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('agentic-recipe-coordinator')

    const createdPods = mockCoreApi.createNamespacedPod.mock.calls.map(call => call[0].body)
    const artifactReaderPod = createdPods.find(
      pod => pod.metadata.name === 'agentic-recipe-artifact-reader'
    )
    const coordinatorPod = createdPods.find(
      pod => pod.metadata.name === 'agentic-recipe-coordinator'
    )
    expect(
      artifactReaderPod.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]
        .labelSelector.matchExpressions
    ).toContainEqual({
      key: 'clerum.io/component',
      operator: 'In',
      values: ['workflow-output-anchor'],
    })
    expect(
      coordinatorPod.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]
        .labelSelector.matchExpressions
    ).toContainEqual({
      key: 'clerum.io/component',
      operator: 'In',
      values: ['workflow-output-anchor'],
    })
  })

  it('replaces a CrashLoopBackOff mcp-host pod before waiting on readiness', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Running' })
    mockMcpHostReadiness({ ready: true, phase: 'Running' })
    crashRecoveryMocks.getContainerWaitingReason.mockImplementation(async (_api, name: string) =>
      name === 'agentic-recipe-mcp-host' ? 'CrashLoopBackOff' : undefined
    )
    crashRecoveryMocks.evaluateCrashRecovery.mockReturnValueOnce({
      action: 'replace',
      message: 'Pod in Running with CrashLoopBackOff; creating replacement (attempt 1/3)',
      newPhase: 'recovering',
      newAttempt: 1,
    })
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing', attempt: 0 } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(crashRecoveryMocks.evaluateCrashRecovery).toHaveBeenCalledWith(
      'Running',
      { phase: 'initializing', attempt: 0 },
      'CrashLoopBackOff'
    )
    expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
      mockCoreApi,
      'agentic-recipe-mcp-host',
      sandboxNamespace
    )
    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('agentic-recipe-mcp-host')
    expect(podNames).toContain('agentic-recipe-coordinator')
  })

  it('replaces a completed mcp-host pod before waiting on readiness', async () => {
    mockAgenticOutputPodPhases({ mcpHost: 'Succeeded' })
    mockMcpHostReadiness({ ready: true, phase: 'Running' })
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.reconcile(
      'agentic-recipe',
      'uid-agentic',
      sandboxNamespace,
      {
        agent: { provider: 'openai', model: 'gpt-4.1' },
        steps: [{ id: 'brief', instruction: 'write the brief' }],
      },
      { workflowExecution: { phase: 'initializing', attempt: 0 } },
      undefined,
      'agentic-recipe',
      'run-1'
    )

    expect(crashRecoveryMocks.evaluateCompletedRuntimePodRecovery).toHaveBeenCalledWith(
      { phase: 'initializing', attempt: 0 },
      'mcp_host'
    )
    expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
      mockCoreApi,
      'agentic-recipe-mcp-host',
      sandboxNamespace
    )
    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('agentic-recipe-mcp-host')
    expect(podNames).toContain('agentic-recipe-coordinator')
  })

  describe('teardownComputePodsForTerminalRun', () => {
    it('deletes coordinator, mcp-host, and snippet-runner but PRESERVES artifact-reader', async () => {
      const reconciler = new WorkflowReconciler(deps)

      await reconciler.teardownComputePodsForTerminalRun('wf-run-7')

      const deletedPods = crashRecoveryMocks.deletePodIfExists.mock.calls.map(
        call => call[1] as string
      )
      expect(deletedPods).toContain('wf-run-7-coordinator')
      expect(deletedPods).toContain('wf-run-7-mcp-host')
      expect(deletedPods).toContain('wf-run-7-snippet-runner')
      // Artifacts must stay downloadable — artifact-reader is never torn down here.
      expect(deletedPods).not.toContain('wf-run-7-artifact-reader')
      // All deletions target the sandbox namespace.
      for (const call of crashRecoveryMocks.deletePodIfExists.mock.calls) {
        expect(call[2]).toBe(sandboxNamespace)
      }
      expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()
    })

    it('is DELETE-first and 404-safe even when pods are already gone', async () => {
      crashRecoveryMocks.deletePodIfExists.mockResolvedValue(undefined)
      const reconciler = new WorkflowReconciler(deps)

      await reconciler.teardownComputePodsForTerminalRun('wf-run-7')

      // Witness: the teardown ran (three DELETEs) and none of them was
      // preceded by a raw pod GET, which getPodPresence alone would not show.
      expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)
      expect(crashRecoveryMocks.getPodPresence).not.toHaveBeenCalled()
      expect(mockCoreApi.readNamespacedPod).not.toHaveBeenCalled()
    })

    it('is idempotent: a second pass still DELETE-first (404-safe)', async () => {
      const reconciler = new WorkflowReconciler(deps)
      await reconciler.teardownComputePodsForTerminalRun('wf-run-7')
      crashRecoveryMocks.deletePodIfExists.mockClear()
      await expect(
        reconciler.teardownComputePodsForTerminalRun('wf-run-7')
      ).resolves.toBeUndefined()
      expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledTimes(3)
    })
  })
})

describe('WorkflowReconciler — Plugin Workload SDK eager mcp-host', () => {
  const sandboxNamespace = 'sandbox-recipes'
  const mcpServerNamespace = 'mcp-server'

  const mockCoreApi = {
    readNamespacedPod: vi.fn().mockResolvedValue({}),
    readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
    readNamespacedService: vi.fn().mockResolvedValue({}),
    readNamespacedEndpoints: vi.fn().mockRejectedValue({ code: 404 }),
    createNamespacedSecret: vi.fn().mockResolvedValue({}),
    createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
    createNamespacedService: vi.fn().mockResolvedValue({}),
    createNamespacedPod: vi.fn().mockResolvedValue({}),
    patchNamespacedSecret: vi.fn().mockResolvedValue({}),
    deleteNamespacedPod: vi.fn().mockResolvedValue({}),
    deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
    deleteNamespacedService: vi.fn().mockResolvedValue({}),
  }
  const mockCustomApi = {
    patchNamespacedCustomObject: vi.fn().mockResolvedValue({}),
    patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
  }
  const mockNetworkingApi = {
    createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    readNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ metadata: { resourceVersion: '1' } }),
    replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
  }
  const mockModelConfigHandler = {
    configurePluginWorkloadSdkBootstrap: vi.fn().mockResolvedValue({
      status: 202,
      body: {
        configured: true,
        ready: true,
        provider: 'zai',
        model: 'glm-4.7',
        contractVersion: 2,
        policyRevision: 1,
        policyHash: 'a'.repeat(64),
        defaultTargetRef: 'primary-zai',
        defaultProvider: 'zai',
        defaultModel: 'glm-4.7',
        clientNotificationsPolicyReady: true,
        clientNotificationsPolicyState: 'active',
      },
    }),
  }
  const mockTokenFactory = {
    signWrcConfigureToken: vi.fn().mockResolvedValue('wrc-configure-token'),
    signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('stub-token'),
  }
  const mockPluginWorkloadSdkRevocationClient = {
    revoke: vi.fn().mockResolvedValue({ state: 'missing', revoked: 0, fencedInvocations: 0 }),
    finalize: vi.fn(),
  }

  const makeDeps = () =>
    ({
      coreApi: mockCoreApi,
      customApi: mockCustomApi,
      networkingApi: mockNetworkingApi,
      config: {
        coordinatorImage: 'coordinator:test',
        mcpHostImage: 'mcp-host:test',
        wrcEndpoint: 'http://wrc.example/api',
        sandboxNamespace,
        mcpServerNamespace,
        imagePullPolicy: 'IfNotPresent' as const,
        maxWorkflowSteps: 100,
        runtimeTokenTtlSeconds: 3600,
        runtimeTokenRefreshBeforeSeconds: 300,
        pluginWorkloadSdkEnabled: true,
      },
      tokenFactory: mockTokenFactory,
      modelConfigHandler: mockModelConfigHandler,
      pluginWorkloadSdkRevocationClient: mockPluginWorkloadSdkRevocationClient,
    }) as unknown as WorkflowReconcilerDeps

  const sdkBootstrapProof = () => ({
    configured: true,
    ready: true,
    provider: 'zai',
    model: 'glm-4.7',
    contractVersion: 2,
    policyRevision: 1,
    policyHash: 'a'.repeat(64),
    defaultTargetRef: 'primary-zai',
    defaultProvider: 'zai',
    defaultModel: 'glm-4.7',
    clientNotificationsPolicyReady: true,
    clientNotificationsPolicyState: 'active',
  })

  beforeEach(() => {
    vi.clearAllMocks()
    crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodPhase.mockResolvedValue(undefined)
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    crashRecoveryMocks.evaluateCrashRecovery.mockReturnValue({ action: 'none', message: 'healthy' })
    mockCoreApi.createNamespacedPod.mockResolvedValue({})
    mockCoreApi.readNamespacedService.mockRejectedValue({ code: 404 })
    mockCoreApi.readNamespacedEndpoints.mockRejectedValue({ code: 404 })
    mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
    mockNetworkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
    // clearAllMocks keeps implementations, so a DELETE failure one test
    // injects must not leak into the next teardown.
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockResolvedValue({})
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: sdkBootstrapProof(),
    })
    vi.stubGlobal(
      'fetch',
      vi.fn().mockResolvedValue({ ok: true, status: 204 } as unknown as Response)
    )
  })

  const sdkSpec = (overrides = {}) =>
    ({
      agent: { provider: 'zai', model: 'glm-4.7' },
      steps: [{ id: 'keepalive', instruction: 'ack' }],
      workloads: [{ id: 'sdk-caller', type: 'deployment', image: 'caller:test' }],
      pluginWorkloadSdk: {
        promptBridge: {},
        allowedCallers: ['sdk-caller'],
      },
      ...overrides,
    }) as unknown as Parameters<WorkflowReconciler['reconcile']>[3]

  it('rotates access, refresh, and control tokens when runtime scopes drift', async () => {
    const now = Math.floor(Date.now() / 1000)
    const staleBinding = {
      exp: now + 3600,
      recipeNamespace: sandboxNamespace,
      recipeName: 'sdk-recipe',
      hostRefs: [`${sandboxNamespace}/sdk-recipe`],
      scopes: [],
    }
    const staleGfsBinding = {
      ...staleBinding,
      hostRefs: [`host:3rd:${sandboxNamespace}/sdk-recipe`],
      scopes: ['gfs.read'],
    }
    mockCoreApi.readNamespacedSecret.mockResolvedValueOnce({
      data: {
        'mcp-host-runtime-access-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
          'base64'
        ),
        'mcp-host-runtime-refresh-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
          'base64'
        ),
        'mcp-host-workflow-control-token': Buffer.from(unsignedRuntimeJwt(staleBinding)).toString(
          'base64'
        ),
        'mcp-host-gfs-token': Buffer.from(unsignedRuntimeJwt(staleGfsBinding)).toString('base64'),
      },
    })

    const reconciler = new WorkflowReconciler(makeDeps())
    await reconciler.ensureMcpHostRuntimeCredentials(sandboxNamespace, 'sdk-recipe', sdkSpec())

    expect(runtimeTokenIssuerMocks.issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
      sandboxNamespace,
      'sdk-recipe',
      ['plugin-workload-sdk']
    )
    expect(mockCoreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)
    const patch = mockCoreApi.patchNamespacedSecret.mock.calls[0]?.[0] as {
      body?: { data?: Record<string, string> }
    }
    expect(patch.body?.data).toEqual(
      expect.objectContaining({
        'mcp-host-runtime-access-token': expect.any(String),
        'mcp-host-runtime-refresh-token': expect.any(String),
        'mcp-host-workflow-control-token': expect.any(String),
      })
    )
  })

  it('creates the mcp-host but NOT the coordinator when no run is triggered', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined // no run id → awaitsTriggeredRun
    )

    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('sdk-recipe-mcp-host')
    expect(podNames).not.toContain('sdk-recipe-coordinator')

    // The eager mcp-host must get the same runtime-token + SDK-token Secrets the
    // triggered-run path provisions, or it boots NotReady (degraded runtime auth).
    const secretNames = mockCoreApi.createNamespacedSecret.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(secretNames).toContain('wf-sdk-recipe-mcp-host-runtime-tokens')
    const runtimeSecret = mockCoreApi.createNamespacedSecret.mock.calls
      .map(call => call[0].body)
      .find(secret => secret.metadata.name === 'wf-sdk-recipe-mcp-host-runtime-tokens')
    expect(runtimeSecret?.stringData?.[['mcp-host-gfs', 'token'].join('-')]).toBe(
      'gfs-runtime-value'
    )

    // The SDK lane NetworkPolicies must also come up so the sdk-caller can
    // reach the SDK server on :8099.
    const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
      call => call[0].body.metadata.name
    )
    // A stepless SDK host has no coordinator. The eager network lane must not
    // create workflow-only coordinator policies that would later be orphaned.
    expect(npNames).not.toContain('sdk-recipe-coord-to-mcp-host')
    expect(npNames).not.toContain('sdk-recipe-coord-to-mcp-host-ingress')
    expect(npNames).not.toContain('sdk-recipe-coord-to-wrc')
    expect(npNames).toContain('sdk-recipe-workload-to-mcp-host-sdk-ingress')
    expect(npNames).toContain('sdk-recipe-workload-to-mcp-host-sdk-egress')
    expect(npNames).toContain('sdk-recipe-mcp-host-to-gfs')

    // The mcp-host headless Service must exist so the SDK endpoint DNS resolves.
    const svcNames = mockCoreApi.createNamespacedService.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(svcNames).toContain('wf-sdk-recipe-mcp-host')
  })

  it('leaves coordinator GFS NetworkPolicy ownership to the outer recipe reconciler', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec({ gfs: { publishTargets: [{ drive: 'main', target: 'outputs' }] } }),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(npNames).not.toContain('sdk-recipe-coordinator-to-gfs')
  })

  it('creates eager mcp-host for SDK-only recipes without agentic steps', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'sdk-only',
      'uid-sdk-only',
      sandboxNamespace,
      sdkSpec({ steps: undefined })
    )

    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('sdk-only-mcp-host')
    expect(podNames).not.toContain('sdk-only-coordinator')
    const npNames = mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(npNames).not.toContain('sdk-only-coord-to-mcp-host')
    expect(npNames).not.toContain('sdk-only-coord-to-mcp-host-ingress')
    expect(npNames).not.toContain('sdk-only-coord-to-wrc')
    expect(result.phase).toBe('active')
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
      'zai',
      'glm-4.7',
      expect.any(String),
      'wrc-configure-token',
      'promptBridge',
      null
    )
  })

  it('reads the eager mcp-host Service and sends no POST when the live one matches', async () => {
    const serviceName = buildMcpHostServiceName('sdk-only')
    const live = {
      ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace),
      metadata: {
        ...buildMcpHostHeadlessService('sdk-only', sandboxNamespace).metadata,
        resourceVersion: 'rv-3',
      },
    }
    mockCoreApi.readNamespacedService.mockImplementation(async ({ name }: { name: string }) => {
      if (name === serviceName) return structuredClone(live)
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(makeDeps())

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'sdk-only',
      'uid-sdk-only',
      sandboxNamespace,
      sdkSpec({ steps: undefined })
    )

    // This fixture has no replaceNamespacedService, so a replace would throw
    // and the phase would not be 'active'.
    expect(result.phase).toBe('active')
    expect(mockCoreApi.readNamespacedService).toHaveBeenCalledWith({
      name: serviceName,
      namespace: sandboxNamespace,
    })
    const createdServiceNames = mockCoreApi.createNamespacedService.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(createdServiceNames).not.toContain(serviceName)
  })

  // The disabled branch applies no policy, so it must not report a summary: an
  // empty summary reads as "evaluated, no conflict" and would clear a published
  // WorkflowNetworkPolicyOwnership condition.
  it('reports no NetworkPolicy summary when the SDK runtime is disabled', async () => {
    const deps = makeDeps()
    const reconciler = new WorkflowReconciler({
      ...deps,
      config: { ...deps.config, pluginWorkloadSdkEnabled: false },
    })

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'sdk-only',
      'uid-sdk-only',
      sandboxNamespace,
      sdkSpec({ steps: undefined })
    )

    expect(result.phase).toBe('active')
    expect(result.message).toBe('Plugin Workload SDK runtime disabled')
    expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
    expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(result.networkPolicies).toBeUndefined()
  })

  describe('NetworkPolicy ownership conflicts', () => {
    const foreignOwnedPolicy = (name: string) => ({
      metadata: {
        name,
        namespace: sandboxNamespace,
        resourceVersion: '7',
        ownerReferences: [
          {
            apiVersion: 'apps/v1',
            kind: 'Deployment',
            name: 'another-controller',
            uid: 'foreign-owner-uid',
            controller: true,
          },
        ],
      },
      spec: { podSelector: {} },
    })
    const foreignOnly = (foreignName: string) =>
      mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
        async ({ name }: { name: string }) => {
          if (name === foreignName) return foreignOwnedPolicy(foreignName)
          throw { code: 404 }
        }
      )
    const writesTo = (name: string) => [
      ...mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.filter(
        call => call[0].body.metadata.name === name
      ),
      ...mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
        call => call[0].name === name
      ),
    ]
    const createdNames = () =>
      mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.map(
        call => call[0].body.metadata.name
      )
    const ownershipCondition = (conditions: unknown) =>
      (conditions as Array<{ type: string }> | undefined)?.find(
        condition => condition.type === 'WorkflowNetworkPolicyOwnership'
      )

    it('SDK-only lane returns the conflict and leaves the foreign policy untouched', async () => {
      const foreignName = 'sdk-only-workload-to-mcp-host-sdk-ingress'
      foreignOnly(foreignName)
      const reconciler = new WorkflowReconciler(makeDeps())

      const result = await reconciler.reconcilePluginWorkloadSdkOnly(
        'sdk-only',
        'uid-sdk-only',
        sandboxNamespace,
        sdkSpec({ steps: undefined })
      )

      expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
        expect.objectContaining({ name: foreignName })
      )
      // The conflict did not stop the rest of the pass.
      expect(createdNames()).toContain('sdk-only-workload-to-mcp-host-sdk-egress')
      expect(result.phase).toBe('active')
      // The eager apply prunes too, so its summary carries the prune and legacy facts.
      expect(result.networkPolicies).toEqual({
        conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
        retryPending: false,
        prune: 'converged',
        legacy: 'removed',
      })
      expect(writesTo(foreignName)).toHaveLength(0)
    })

    // Every eager host status that is not `ready` still ran the apply when it
    // carries a summary, so the SDK-only lane must hand that summary on: dropping
    // it would read as "returned before applying" and keep a stale condition.
    it.each([
      { status: 'awaiting_policy', phase: 'awaiting_policy' },
      { status: 'deploying', phase: 'deploying' },
      { status: 'provider_unavailable', phase: 'provider_unavailable' },
      { status: 'failed', phase: 'failed' },
    ] as const)(
      'SDK-only lane passes the eager apply summary through on $status',
      async ({ status, phase }) => {
        const reconciler = new WorkflowReconciler(makeDeps())
        const provisioner = (
          reconciler as unknown as {
            pluginWorkloadSdkProvisioner: {
              ensureEagerSdkMcpHost: (...args: unknown[]) => unknown
            }
          }
        ).pluginWorkloadSdkProvisioner
        const foreignName = 'sdk-only-workload-to-mcp-host-sdk-ingress'
        const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
          status,
          networkPolicies: {
            conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
            retryPending: true,
            prune: 'converged',
            legacy: 'removed',
          },
        })

        const result = await reconciler.reconcilePluginWorkloadSdkOnly(
          'sdk-only',
          'uid-sdk-only',
          sandboxNamespace,
          sdkSpec({ steps: undefined })
        )

        expect(ensure).toHaveBeenCalledTimes(1)
        expect(ensure.mock.calls[0]?.[0]).toBe('sdk-only')
        expect(result.phase).toBe(phase)
        expect(result.networkPolicies).toEqual({
          conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
          retryPending: true,
          prune: 'converged',
          legacy: 'removed',
        })
      }
    )

    it('workflow lane surfaces the conflict as a False condition and clears it on recovery', async () => {
      const foreignName = 'sdk-recipe-workload-to-mcp-host-sdk-ingress'
      foreignOnly(foreignName)
      const reconciler = new WorkflowReconciler(makeDeps())

      const first = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledWith(
        expect.objectContaining({ name: foreignName })
      )
      expect(createdNames()).toContain('sdk-recipe-workload-to-mcp-host-sdk-egress')
      expect(first.phase).not.toBe('failed')
      expect(first.workflowPhase).not.toBe('failed')
      expect(first.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPolicyOwnership',
          status: 'False',
          reason: 'OwnershipConflict',
          message: expect.stringContaining(foreignName),
        }),
      ])
      // The condition travels only in its own field, never with the
      // workflow-output conditions that patchStatus merges separately.
      expect(ownershipCondition(first.workflowConditions)).toBeUndefined()
      expect(writesTo(foreignName)).toHaveLength(0)

      // The foreign owner goes away: the policy is created and the owned set
      // is empty, so patchStatus removes the condition.
      mockNetworkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
      mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
      const second = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        {
          workflowExecution: { phase: 'initializing' },
          conditions: first.networkPolicyOwnershipConditions,
        },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(createdNames()).toContain(foreignName)
      expect(second.networkPolicyOwnershipConditions).toEqual([])
    })

    // The pass never reached the apply, so it cannot say whether the conflict is
    // gone. The field stays undefined and patchStatus keeps the published one.
    it('workflow lane leaves the ownership field undefined when the eager host fails before applying', async () => {
      const reconciler = new WorkflowReconciler(makeDeps())
      const provisioner = (
        reconciler as unknown as {
          pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
        }
      ).pluginWorkloadSdkProvisioner
      const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
        status: 'failed',
      })
      const existing = {
        type: 'WorkflowNetworkPolicyOwnership',
        status: 'False' as const,
        reason: 'OwnershipConflict',
        message: 'sdk-recipe-mcp-host-to-gfs (owner-reference-mismatch)',
        lastTransitionTime: '2026-09-23T10:00:00.000Z',
      }

      const result = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' }, conditions: [existing] },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(ensure).toHaveBeenCalledTimes(1)
      expect(result.phase).toBe('failed')
      expect(result).not.toHaveProperty('networkPolicyOwnershipConditions')
      expect(ownershipCondition(result.workflowConditions)).toBeUndefined()
    })

    // The pod failed after the policies were applied, so the summary is a real
    // evaluation and the conflict it found is published with the failure.
    it('workflow lane publishes the evaluated set when the eager host fails after applying', async () => {
      const reconciler = new WorkflowReconciler(makeDeps())
      const provisioner = (
        reconciler as unknown as {
          pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
        }
      ).pluginWorkloadSdkProvisioner
      const foreignName = 'sdk-recipe-mcp-host-to-gfs'
      const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
        status: 'failed',
        networkPolicies: {
          conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
          retryPending: false,
          prune: 'converged',
          legacy: 'removed',
        },
      })

      const result = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(ensure).toHaveBeenCalledTimes(1)
      expect(result.phase).toBe('failed')
      expect(result.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPolicyOwnership',
          status: 'False',
          reason: 'OwnershipConflict',
          message: expect.stringContaining(`${foreignName} (owner-reference-mismatch)`),
        }),
      ])
    })

    // The early return skips the run-lane apply, so the eager summary is the
    // only source of the retry flag for this pass.
    it('workflow lane asks for a retry when the eager host fails after a deferred policy', async () => {
      const reconciler = new WorkflowReconciler(makeDeps())
      const provisioner = (
        reconciler as unknown as {
          pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
        }
      ).pluginWorkloadSdkProvisioner
      const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
        status: 'failed',
        networkPolicies: {
          conflicts: [],
          retryPending: true,
          prune: 'converged',
          legacy: 'removed',
        },
      })

      const result = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(ensure).toHaveBeenCalledTimes(1)
      expect(result.phase).toBe('failed')
      expect(result.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPoliciesConverged',
          status: 'False',
          reason: 'RetryPending',
        }),
      ])
      expect(result.networkPolicyRetryPending).toBe(true)
    })

    // provider_unavailable overrides workflowConditions after the spread, so it
    // must still carry the ownership set and the retry flag from the eager apply.
    it('workflow lane publishes the eager apply summary on provider_unavailable', async () => {
      const reconciler = new WorkflowReconciler(makeDeps())
      const provisioner = (
        reconciler as unknown as {
          pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
        }
      ).pluginWorkloadSdkProvisioner
      const foreignName = 'sdk-recipe-mcp-host-to-gfs'
      const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
        status: 'provider_unavailable',
        networkPolicies: {
          conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
          retryPending: true,
          prune: 'converged',
          legacy: 'removed',
        },
      })

      const result = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(ensure).toHaveBeenCalledTimes(1)
      expect(result.phase).toBe('active')
      expect(result.workflowConditions).toEqual(
        expect.arrayContaining([
          expect.objectContaining({ type: 'PluginWorkloadSdkProviderUnavailable' }),
        ])
      )
      expect(result.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPolicyOwnership',
          status: 'False',
          reason: 'OwnershipConflict',
          message: expect.stringContaining(`${foreignName} (owner-reference-mismatch)`),
        }),
        expect.objectContaining({
          type: 'WorkflowNetworkPoliciesConverged',
          status: 'False',
          reason: 'RetryPending',
        }),
      ])
      expect(result.networkPolicyRetryPending).toBe(true)
    })

    // A conflict that persists across passes keeps the time it was first seen,
    // so the condition does not look like a new transition on every pass.
    it('workflow lane keeps the first transition time of a persisting conflict', async () => {
      const reconciler = new WorkflowReconciler(makeDeps())
      const provisioner = (
        reconciler as unknown as {
          pluginWorkloadSdkProvisioner: { ensureEagerSdkMcpHost: (...args: unknown[]) => unknown }
        }
      ).pluginWorkloadSdkProvisioner
      const foreignName = 'sdk-recipe-mcp-host-to-gfs'
      const firstSeen = '2026-09-23T10:00:00.000Z'
      const ensure = vi.spyOn(provisioner, 'ensureEagerSdkMcpHost').mockResolvedValue({
        status: 'failed',
        networkPolicies: {
          conflicts: [{ policy: foreignName, reason: 'owner-reference-mismatch' }],
          retryPending: false,
          prune: 'converged',
          legacy: 'removed',
        },
      })

      const result = await reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        {
          workflowExecution: { phase: 'initializing' },
          conditions: [
            {
              type: 'WorkflowNetworkPolicyOwnership',
              status: 'False',
              reason: 'OwnershipConflict',
              message: `NetworkPolicy ownership conflict: ${foreignName} (owner-reference-mismatch)`,
              lastTransitionTime: firstSeen,
            },
          ],
        },
        undefined,
        'sdk-recipe',
        undefined
      )

      expect(ensure).toHaveBeenCalledTimes(1)
      expect(result.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPolicyOwnership',
          status: 'False',
          message: expect.stringContaining(foreignName),
          lastTransitionTime: firstSeen,
        }),
      ])
    })
  })

  it('creates a provider-free eager mcp-host for clientNotifications-only without an agent', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: {
        configured: true,
        ready: true,
        contractVersion: 2,
        capabilityFamily: 'clientNotifications',
        policyReady: true,
        policyState: 'active',
      },
    })

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'notifications-only',
      'uid-notifications-only',
      sandboxNamespace,
      sdkSpec({
        agent: undefined,
        steps: undefined,
        pluginWorkloadSdk: {
          clientNotifications: { allowedEventTypes: ['e2e.test'] },
          allowedCallers: ['sdk-caller'],
        },
      })
    )

    expect(result.phase).toBe('active')
    const pod = mockCoreApi.createNamespacedPod.mock.calls[0]?.[0].body
    const envNames = (pod.spec.containers[0].env ?? []).map((entry: { name: string }) => entry.name)
    expect(envNames).not.toContain('CLERUM_MODEL_PROVIDER')
    expect(envNames).not.toContain('CLERUM_MODEL')
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
      undefined,
      undefined,
      expect.any(String),
      'wrc-configure-token',
      'clientNotifications'
    )
  })

  it('cleans SDK-only host resources idempotently without creating workflow resources', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()

    expect(mockCoreApi.deleteNamespacedService).toHaveBeenCalledWith({
      name: 'wf-sdk-only-mcp-host',
      namespace: sandboxNamespace,
    })
    expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
      name: 'wf-sdk-only-plugin-workload-sdk-token',
      namespace: sandboxNamespace,
    })
    expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
      name: 'wf-sdk-only-mcp-host-runtime-tokens',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'sdk-only-workload-to-mcp-host-sdk-ingress',
      namespace: sandboxNamespace,
    })
    expect(mockCoreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  // R4-L3: the SDK-only teardown is what makes the `unmanaged` marker writer
  // true. The legacy mcp-servers internet policy may carry no labels, so the
  // label sweep cannot be trusted to remove it; the teardown deletes it by name.
  it('R4-L3: deletes the legacy internet policy by name in the SDK-only teardown, next to the label sweep', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()

    // Liveness witness: the label sweep ran in both namespaces.
    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: sandboxNamespace,
      labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
    })
    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: mcpServerNamespace,
      labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'sdk-only-mcp-servers-egress-internet',
      namespace: mcpServerNamespace,
    })
  })

  it('fails live SDK cleanup with the remaining shared legacy-delete window', async () => {
    vi.useFakeTimers({ toFake: ['Date'] })
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
      async ({ name }: { name: string }) => {
        if (name === 'sdk-only-mcp-servers-egress-internet') {
          throw { code: 403, message: 'forbidden' }
        }
        return {}
      }
    )
    const reconciler = new WorkflowReconciler(makeDeps())

    try {
      const pending = await reconciler
        .cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
        .catch(error => error)

      expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
      expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(
        LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS
      )
      // Positive API witness: the shared legacy path attempted the DELETE;
      // the typed error describes the live cleanup's remaining backoff.
      expect(
        mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
        )
      ).toHaveLength(1)
    } finally {
      vi.useRealTimers()
    }
  })

  // A-1: a pending legacy mcp-servers internet DELETE must not hold the SDK
  // host pod and token Secrets in place. The legacy policy is unrelated to the
  // SDK credentials, so the pod and Secret deletes run on every pass while the
  // cleanup still fails closed with the legacy-pending error.
  it('A-1: deletes the SDK host pod and token Secrets while the legacy policy DELETE is pending', async () => {
    vi.useFakeTimers({ toFake: ['Date'] })
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
      async ({ name }: { name: string }) => {
        if (name === 'sdk-only-mcp-servers-egress-internet') {
          throw { code: 403, message: 'forbidden' }
        }
        return {}
      }
    )
    const reconciler = new WorkflowReconciler(makeDeps())
    const policyDeletes = (name: string) =>
      mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === name
      )

    try {
      const first = await reconciler
        .cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
        .catch(error => error)
      const second = await reconciler
        .cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
        .catch(error => error)

      expect(first).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
      expect(second).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
      // Liveness witness: both passes ran the SDK network-policy teardown.
      expect(policyDeletes('sdk-only-workload-to-mcp-host-sdk-ingress')).toHaveLength(2)
      // The first pass attempted the legacy DELETE; the second is inside the
      // backoff window and sends no request.
      expect(policyDeletes('sdk-only-mcp-servers-egress-internet')).toHaveLength(1)
      expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledTimes(2)
      expect(mockCoreApi.deleteNamespacedPod).toHaveBeenCalledWith({
        name: 'sdk-only-mcp-host',
        namespace: sandboxNamespace,
        propagationPolicy: 'Background',
      })
      expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledTimes(4)
      expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
        name: 'wf-sdk-only-plugin-workload-sdk-token',
        namespace: sandboxNamespace,
      })
      expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
        name: 'wf-sdk-only-mcp-host-runtime-tokens',
        namespace: sandboxNamespace,
      })
    } finally {
      vi.useRealTimers()
    }
  })

  it('R4-L3: keeps the legacy internet policy in a hybrid teardown, which preserves the workflow runtime', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await expect(
      reconciler.cleanupPluginWorkloadSdk('hybrid', { preserveWorkflowRuntime: true })
    ).resolves.toBeUndefined()

    // Liveness witness: the SDK-owned policies were deleted.
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'hybrid-workload-to-mcp-host-sdk-ingress',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
      name: 'hybrid-mcp-servers-egress-internet',
      namespace: mcpServerNamespace,
    })
  })

  it('uses recipeUid as live identity and preserves the shared legacy gone-set', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())
    const legacyDeletes = () =>
      mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
      )

    await expect(
      reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
    ).resolves.toBe('removed')
    expect(legacyDeletes()).toHaveLength(1)

    await reconciler.cleanupPluginWorkloadSdk('sdk-only', { recipeUid: 'uid-sdk-only' })
    expect(legacyDeletes()).toHaveLength(1)

    // This old-UID retry is a direct ledger probe. It does not pretend that
    // Kubernetes recreated an object with the old UID.
    await expect(
      reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
    ).resolves.toBe('removed')
    expect(legacyDeletes()).toHaveLength(1)
  })

  it('gives finalizer cleanup one last chance and forgets only after it completes', async () => {
    vi.useFakeTimers({ toFake: ['Date'] })
    const reconciler = new WorkflowReconciler(makeDeps())
    const legacyDeletes = () =>
      mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
      )
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
      async ({ name }: { name: string }) => {
        if (name === 'sdk-only-mcp-servers-egress-internet') {
          throw { code: 403, message: 'forbidden' }
        }
        return {}
      }
    )

    try {
      await expect(
        reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
      ).resolves.toBe('pending')
      expect(legacyDeletes()).toHaveLength(1)

      // Finalizer cleanup must attempt the DELETE even inside the live window.
      const pending = await reconciler
        .cleanupPluginWorkloadSdk('sdk-only', {
          recipeUid: 'uid-sdk-only',
          recipeDeleted: true,
        })
        .catch(error => error)
      expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
      expect(legacyDeletes()).toHaveLength(2)

      // A normal old-UID probe is still inside the shared backoff. This direct
      // ledger probe does not model a Kubernetes object recreation.
      await expect(
        reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
      ).resolves.toBe('pending')
      expect(legacyDeletes()).toHaveLength(2)

      mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async () => ({}))
      await reconciler.cleanupPluginWorkloadSdk('sdk-only', {
        recipeUid: 'uid-sdk-only',
        recipeDeleted: true,
      })
      expect(legacyDeletes()).toHaveLength(3)

      // The successful, completed finalizer cleanup forgot the old entry. The
      // old-UID probe is again a direct process-local ledger witness.
      await expect(
        reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
      ).resolves.toBe('removed')
      expect(legacyDeletes()).toHaveLength(4)
    } finally {
      vi.useRealTimers()
    }
  })

  it('does not forget the finalizer ledger entry when a later cleanup phase fails', async () => {
    const lateError = new Error('late finalize failed')
    const revocationId = 'late-finalize-revocation'
    const deps = makeDeps() as unknown as {
      pluginWorkloadSdkRevocationClient?: unknown
    }
    const revocationClient = {
      revoke: vi.fn().mockResolvedValue({
        state: 'revoking',
        revocationId,
        revoked: 1,
        fencedInvocations: 1,
      }),
      finalize: vi.fn().mockRejectedValue(lateError),
    }
    deps.pluginWorkloadSdkRevocationClient = revocationClient
    const reconciler = new WorkflowReconciler(deps as WorkflowReconcilerDeps)
    const legacyDeletes = () =>
      mockNetworkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === 'sdk-only-mcp-servers-egress-internet'
      )

    await expect(
      reconciler.cleanupPluginWorkloadSdk('sdk-only', {
        recipeUid: 'uid-sdk-only',
        recipeDeleted: true,
      })
    ).rejects.toBe(lateError)

    // Positive witnesses: the central legacy DELETE succeeded, and the later
    // revocation-finalization phase—not an earlier network failure—rejected.
    expect(legacyDeletes()).toHaveLength(1)
    expect(revocationClient.finalize).toHaveBeenCalledWith(
      sandboxNamespace,
      'sdk-only',
      revocationId
    )

    // The old-UID retry is a direct process-local ledger probe, not a
    // Kubernetes recreation. It must hit the recorded gone-set, not send a
    // second DELETE after only partial cleanup completion.
    await expect(
      reconciler.retryLegacyMcpServersInternetEgressDelete('sdk-only', 'uid-sdk-only')
    ).resolves.toBe('removed')
    expect(legacyDeletes()).toHaveLength(1)
  })

  it('defers absence reads until pod deletion completes so 404s cannot become unhandled rejections', async () => {
    const events: string[] = []
    crashRecoveryMocks.waitForPodDeletion.mockImplementationOnce(async () => {
      events.push('wait-start')
      await Promise.resolve()
      events.push('wait-end')
      return true
    })
    mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async () => {
      events.push('network-policy-read')
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(makeDeps())

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()

    expect(events.indexOf('wait-start')).toBeGreaterThanOrEqual(0)
    expect(events.indexOf('wait-end')).toBeGreaterThan(events.indexOf('wait-start'))
    expect(events.indexOf('network-policy-read')).toBeGreaterThan(events.indexOf('wait-end'))
  })

  it('removes only SDK-owned authority from a hybrid workflow runtime', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await expect(
      reconciler.cleanupPluginWorkloadSdk('hybrid', { preserveWorkflowRuntime: true })
    ).resolves.toBeUndefined()

    expect(mockCoreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
      name: 'wf-hybrid-plugin-workload-sdk-token',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'hybrid-workload-to-mcp-host-sdk-ingress',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'hybrid-workload-to-mcp-host-sdk-egress',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'hybrid-mcp-host-to-wrc-sdk-broker',
      namespace: sandboxNamespace,
    })
    expect(mockCoreApi.deleteNamespacedService).not.toHaveBeenCalled()
    expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
    expect(mockCoreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
      name: 'wf-hybrid-mcp-host-runtime-tokens',
      namespace: sandboxNamespace,
    })
    expect(mockNetworkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })

  it('fails closed before physical cleanup when the Control API revocation client is absent', async () => {
    const deps = makeDeps() as unknown as { pluginWorkloadSdkRevocationClient?: unknown }
    delete deps.pluginWorkloadSdkRevocationClient
    const reconciler = new WorkflowReconciler(deps as unknown as WorkflowReconcilerDeps)

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
      /revocation client is not configured/
    )
    expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
  })

  it('retains every SDK resource when Control API revocation fails', async () => {
    const deps = makeDeps()
    mockPluginWorkloadSdkRevocationClient.revoke.mockRejectedValueOnce(
      new Error('control-api unavailable')
    )
    const reconciler = new WorkflowReconciler(deps)

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
      'control-api unavailable'
    )
    expect(mockCoreApi.deleteNamespacedPod).not.toHaveBeenCalled()
    expect(mockCoreApi.deleteNamespacedService).not.toHaveBeenCalled()
    expect(mockCoreApi.deleteNamespacedSecret).not.toHaveBeenCalled()
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })

  it('converges after a post-revocation delete failure on the next reconcile', async () => {
    const deps = makeDeps()
    const revocationId = '55555555-5555-4555-8555-555555555555'
    mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
      state: 'revoking',
      revocationId,
      revoked: 1,
      fencedInvocations: 1,
    })
    mockPluginWorkloadSdkRevocationClient.finalize.mockResolvedValue({
      state: 'disabled',
      revocationId,
      revoked: 0,
      fencedInvocations: 0,
      disabled: 1,
    })
    mockCoreApi.deleteNamespacedService.mockRejectedValueOnce(new Error('apiserver timeout'))
    const reconciler = new WorkflowReconciler(deps)

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
      'apiserver timeout'
    )
    expect(mockPluginWorkloadSdkRevocationClient.finalize).not.toHaveBeenCalled()

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
    expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledWith(
      sandboxNamespace,
      'sdk-only',
      revocationId
    )
  })

  it('retries finalization after a transient Control API failure once resources are absent', async () => {
    const deps = makeDeps()
    const revocationId = '66666666-6666-4666-8666-666666666666'
    mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
      state: 'revoking',
      revocationId,
      revoked: 1,
      fencedInvocations: 0,
    })
    mockPluginWorkloadSdkRevocationClient.finalize
      .mockRejectedValueOnce(new Error('control-api finalize timeout'))
      .mockResolvedValueOnce({
        state: 'disabled',
        revocationId,
        revoked: 0,
        fencedInvocations: 0,
        disabled: 1,
      })
    const reconciler = new WorkflowReconciler(deps)

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).rejects.toThrow(
      'control-api finalize timeout'
    )
    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
    expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledTimes(2)
    expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenLastCalledWith(
      sandboxNamespace,
      'sdk-only',
      revocationId
    )
  })

  it('accepts an already-removed grant as terminal revocation confirmation', async () => {
    const deps = makeDeps()
    const revocationId = '77777777-7777-4777-8777-777777777777'
    mockPluginWorkloadSdkRevocationClient.revoke.mockResolvedValue({
      state: 'revoking',
      revocationId,
      revoked: 1,
      fencedInvocations: 0,
    })
    mockPluginWorkloadSdkRevocationClient.finalize.mockResolvedValue({
      state: 'missing',
      revoked: 0,
      fencedInvocations: 0,
    })
    const reconciler = new WorkflowReconciler(deps)

    await expect(reconciler.cleanupPluginWorkloadSdk('sdk-only')).resolves.toBeUndefined()
    expect(mockPluginWorkloadSdkRevocationClient.finalize).toHaveBeenCalledWith(
      sandboxNamespace,
      'sdk-only',
      revocationId
    )
  })

  it('sweeps legacy SDK network policies by recipe label during cleanup', async () => {
    let legacyPolicyPresent = true
    mockNetworkingApi.listNamespacedNetworkPolicy.mockImplementation(
      ({ namespace }: { namespace: string }) =>
        Promise.resolve({
          items:
            namespace === sandboxNamespace && legacyPolicyPresent
              ? [{ metadata: { name: 'sdk-only-legacy-coord-to-mcp-host' } }]
              : [],
        })
    )
    mockNetworkingApi.deleteNamespacedNetworkPolicy.mockImplementation(
      async ({ name }: { name?: string }) => {
        if (name === 'sdk-only-legacy-coord-to-mcp-host') legacyPolicyPresent = false
        return {}
      }
    )
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.cleanupPluginWorkloadSdk('sdk-only')

    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: sandboxNamespace,
      labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
    })
    expect(mockNetworkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
      namespace: mcpServerNamespace,
      labelSelector: 'clerum.io/recipe=sdk-only,clerum.io/managed-by=wrc',
    })
    expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'sdk-only-legacy-coord-to-mcp-host',
      namespace: sandboxNamespace,
    })
  })

  it('brokers the provider into the eager mcp-host for promptBridge recipes', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(1)
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
      'zai',
      'glm-4.7',
      expect.any(String),
      'wrc-configure-token',
      'promptBridge',
      null
    )
  })

  it('does not broker a provider for clientNotifications-only recipes but proves its grant', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: {
        configured: true,
        ready: true,
        contractVersion: 2,
        capabilityFamily: 'clientNotifications',
        policyReady: true,
        policyState: 'active',
      },
    })

    await reconciler.reconcile(
      'notif-recipe',
      'uid-notif',
      sandboxNamespace,
      sdkSpec({
        pluginWorkloadSdk: {
          clientNotifications: { allowedEventTypes: ['e2e.test'] },
          allowedCallers: ['sdk-caller'],
        },
      }),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'notif-recipe',
      undefined
    )

    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('notif-recipe-mcp-host')
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledWith(
      undefined,
      undefined,
      expect.any(String),
      'wrc-configure-token',
      'clientNotifications'
    )
  })

  it('keeps clientNotifications-only recipes awaiting policy when the host proves no grant', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: {
        configured: true,
        ready: true,
        contractVersion: 2,
        capabilityFamily: 'clientNotifications',
        policyReady: false,
        policyState: 'missing',
        policyReason: 'grant_missing',
      },
    })

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'notifications-policy-pending',
      'uid-notifications-policy-pending',
      sandboxNamespace,
      sdkSpec({
        agent: undefined,
        steps: undefined,
        pluginWorkloadSdk: {
          clientNotifications: { allowedEventTypes: ['e2e.test'] },
          allowedCallers: ['sdk-caller'],
        },
      })
    )

    expect(result.phase).toBe('awaiting_policy')
    expect(result.message).toContain('operator policy pending')
    expect(result.message).toContain('grant_missing')
  })

  it('skips the provider broker while the eager mcp-host is not ready', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({ ready: false, phase: 'Pending' })
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).not.toHaveBeenCalled()
  })

  it('returns phase=deploying while the eager mcp-host boots so the watcher requeues the configure', async () => {
    // The eager pod is created but not Ready yet, so /configure is skipped this
    // pass. The result MUST be phase=deploying (NOT active) so the watcher
    // requeues at the fixed progress interval and re-runs the eager /configure
    // once the pod is Ready. Returning 'active' drops the requeue
    // (requeueAfterMs=undefined) — the now-Ready pod is never re-reconciled and
    // stays "waiting for /configure" forever, so promptBridge / clientNotifications
    // fail with provider_unavailable. Regression guard for the readiness race.
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({ ready: false, phase: 'Pending' })
    const reconciler = new WorkflowReconciler(makeDeps())

    const result = await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(result.phase).toBe('deploying')
    expect(result.message).toContain('starting')
    expect(result.message).not.toContain('registered')
  })

  it('returns phase=active once the eager mcp-host is Ready and configured (steady state)', async () => {
    // Pod Ready + /configure succeeds → eagerStatus='ready'. The recipe is
    // settled: phase=active (no progress requeue); the workflowNeedsInfrastructure
    // Reconcile keep-alive loop handles steady-state re-configure/self-heal.
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    const reconciler = new WorkflowReconciler(makeDeps())

    const result = await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(result.phase).toBe('active')
    expect(result.message).toContain('registered')
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(1)
  })

  it('reports awaiting_policy when identity bootstrap is ready but the operator grant is missing', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: {
        configured: true,
        ready: true,
        provider: 'zai',
        model: 'glm-4.7',
        contractVersion: 2,
        policyReady: false,
        policyState: 'missing',
        policyReason: 'grant_missing',
      },
    })
    const reconciler = new WorkflowReconciler(makeDeps())

    const result = await reconciler.reconcilePluginWorkloadSdkOnly(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec({ steps: undefined })
    )

    expect(result.phase).toBe('awaiting_policy')
    expect(result.message).toContain('operator policy pending')
    expect(result.message).toContain('grant_missing')
    expect(result.message).not.toContain('provider unavailable')
    expect(result.pluginWorkloadSdkBootstrapProof).toMatchObject({
      ready: true,
      policyReady: false,
      policyState: 'missing',
      policyReason: 'grant_missing',
    })
  })

  it('re-probes live bootstrap policy on every reconcile of the same ready pod', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    const reconciler = new WorkflowReconciler(makeDeps())
    const run = () =>
      reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

    await run()
    await run()

    // A pod UID is not proof that the operator policy is unchanged. Each
    // reconcile must obtain a fresh Control API proof so grant revocation and
    // target-order changes become effective without restarting the pod.
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
  })

  it('stays pending (not registered) and retries when the provider broker fails', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 500,
      body: { error: 'boom' },
    })
    const reconciler = new WorkflowReconciler(makeDeps())
    const run = () =>
      reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

    const first = await run()
    // Configure failed → recipe must NOT report the SDK as registered/ready.
    expect(first.message).toContain('starting')
    expect(first.message).not.toContain('registered')

    // Failure must not poison the configure cache: the next reconcile retries.
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: sdkBootstrapProof(),
    })
    const second = await run()
    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
    expect(second.message).toContain('registered')
  })

  it('does not publish SDK readiness when the bootstrap contract is not v2', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: { configured: true },
    })
    const reconciler = new WorkflowReconciler(makeDeps())
    const result = await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(result.phase).toBe('deploying')
    expect(result.message).toContain('starting')
    expect(result.message).not.toContain('registered')
  })

  it('projects provider_unavailable after repeated configure failures instead of an indefinite starting', async () => {
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    // Provider broker is permanently broken (e.g. bad secretRef).
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 500,
      body: { error: 'boom' },
    })
    const reconciler = new WorkflowReconciler(makeDeps())
    const run = () =>
      reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

    // First two failures stay "starting" so transient blips keep retrying.
    const first = await run()
    expect(first.message).toContain('starting')
    expect(first.message).not.toContain('provider unavailable')
    const second = await run()
    expect(second.message).toContain('starting')

    // Third consecutive failure crosses MAX_EAGER_CONFIGURE_ATTEMPTS → distinct
    // provider_unavailable projection (still phase=active so it self-heals).
    const third = await run()
    expect(third.phase).toBe('active')
    expect(third.message).toContain('provider unavailable')
    expect(third.workflowConditions).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          type: 'PluginWorkloadSdkProviderUnavailable',
          reason: 'EagerConfigureFailed',
          status: 'True',
        }),
      ])
    )

    // It keeps retrying — recovers as soon as the broker config is fixed, and
    // the unavailable condition is no longer projected.
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 202,
      body: sdkBootstrapProof(),
    })
    const recovered = await run()
    expect(recovered.message).toContain('registered')
    expect(recovered.workflowConditions ?? []).not.toEqual(
      expect.arrayContaining([
        expect.objectContaining({ type: 'PluginWorkloadSdkProviderUnavailable' }),
      ])
    )
  })

  it('restarts the configure failure budget when the eager mcp-host pod is externally replaced', async () => {
    let podUid = 'pod-uid-1'
    crashRecoveryMocks.getPodReadiness.mockImplementation(async (_api, podName: string) => {
      if (podName.endsWith('-mcp-host')) {
        return { ready: true, phase: 'Running', uid: podUid }
      }
      return { ready: true, phase: 'Running' }
    })
    mockModelConfigHandler.configurePluginWorkloadSdkBootstrap.mockResolvedValue({
      status: 500,
      body: { error: 'boom' },
    })
    const reconciler = new WorkflowReconciler(makeDeps())
    const run = () =>
      reconciler.reconcile(
        'sdk-recipe',
        'uid-sdk',
        sandboxNamespace,
        sdkSpec(),
        { workflowExecution: { phase: 'initializing' } },
        undefined,
        'sdk-recipe',
        undefined
      )

    expect((await run()).message).toContain('starting')
    expect((await run()).message).toContain('starting')

    // The replacement was not rolled by WRC, so no explicit reset hook runs.
    // The budget must still restart because it is keyed by pod UID/provider/model.
    podUid = 'pod-uid-2'
    const replacementFirstFailure = await run()
    expect(replacementFirstFailure.message).toContain('starting')
    expect(replacementFirstFailure.message).not.toContain('provider unavailable')

    expect((await run()).message).toContain('starting')
    const replacementThirdFailure = await run()
    expect(replacementThirdFailure.message).toContain('provider unavailable')
  })

  it('re-brokers the provider after the eager mcp-host pod restarts (new uid)', async () => {
    const reconciler = new WorkflowReconciler(makeDeps())
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-1',
    })
    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )
    // Pod restarted — new UID invalidates the cache, forcing a fresh configure.
    crashRecoveryMocks.getPodReadiness.mockResolvedValue({
      ready: true,
      phase: 'Running',
      uid: 'pod-uid-2',
    })
    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(mockModelConfigHandler.configurePluginWorkloadSdkBootstrap).toHaveBeenCalledTimes(2)
  })

  it('recreates a wedged (CrashLoopBackOff) eager mcp-host pod', async () => {
    // Pod exists and is Running but stuck in CrashLoopBackOff — the terminal-phase
    // guard alone would never replace it, so the eager path must delete it first.
    crashRecoveryMocks.getPodPhase.mockResolvedValue('Running')
    crashRecoveryMocks.getContainerWaitingReason.mockResolvedValue('CrashLoopBackOff')
    const reconciler = new WorkflowReconciler(makeDeps())

    await reconciler.reconcile(
      'sdk-recipe',
      'uid-sdk',
      sandboxNamespace,
      sdkSpec(),
      { workflowExecution: { phase: 'initializing' } },
      undefined,
      'sdk-recipe',
      undefined
    )

    expect(crashRecoveryMocks.deletePodIfExists).toHaveBeenCalledWith(
      mockCoreApi,
      'sdk-recipe-mcp-host',
      sandboxNamespace
    )
    // After deletion the pod is recreated in the same pass.
    const podNames = mockCoreApi.createNamespacedPod.mock.calls.map(
      call => call[0].body.metadata.name
    )
    expect(podNames).toContain('sdk-recipe-mcp-host')
  })
})