Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
375 KB
Raw Blame
/**
 * Tests for workflowReconciler reconcile loop.
 *  *
 * WorkflowReconciler.reconcile() signature:
 *   reconcile(recipeName, recipeUid, namespace, spec, currentStatus?)
 *
 * Deps: { coreApi, customApi, networkingApi, batchApi?, config, tokenFactory }
 */
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import type { CodexExecutionProjection } from '@clerum/codex-catalog-projection'
import { mintRecipeHostGfsToken } from '../../../src/gfsBinding'
import { captureLogger } from '../../../src/reconciler/__tests__/captureLogger'
import {
  resolveStatefulSetHeadlessServiceName,
  resolveWorkloadMcpServerLabel,
  resolveWorkloadRuntimeResourceName,
} from '../../../src/reconciler/resourceBuilder'
import {
  issueMcpHostRuntimeTokens,
  issueMcpHostWorkflowControlToken,
} from '../../../src/workflow/mcpHostRuntimeTokenIssuerClient'
import { buildWorkflowNetworkPolicies } from '../../../src/workflow/networkPolicyFactory'
import type { NetworkPolicyPassSummary } from '../../../src/workflow/pluginWorkloadSdkProvisioner'
import {
  buildArtifactReaderHeadlessService,
  buildMcpHostHeadlessService,
  buildMcpHostRouteAliasHeadlessService,
  buildSnippetRunnerHeadlessService,
} from '../../../src/workflow/podFactory'
import {
  buildArtifactReaderServiceName,
  buildMcpHostRouteAliasServiceName,
  buildMcpHostServiceName,
  buildSnippetRunnerServiceName,
} from '../../../src/workflow/resourceNames'
import { deriveWorkflowRuntimePlan } from '../../../src/workflow/runtimePlan'
import {
  LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS,
  LegacyNetworkPolicyDeletePendingError,
  type WorkflowRecipeSpec,
  WorkflowReconciler,
  type WorkflowReconcilerDeps,
  buildNetworkPolicyConvergedCondition,
  buildNetworkPolicyOwnershipConditions,
  hasLegacyNetworkPolicyDeletePendingMarker,
  networkPolicyConditionsChanged,
  networkPolicyMarkerConditions,
  networkPolicyMarkerFacts,
  translateNetworkPolicyPassSummary,
} from '../../../src/workflow/workflowReconciler'
import { asApiserverNetworkPolicy } from './asApiserverNetworkPolicy'

vi.mock('../../../src/workflow/mcpHostRuntimeTokenIssuerClient', () => ({
  issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
    accessToken: 'mcp-host-runtime-access-token',
    refreshToken: 'mcp-host-runtime-refresh-token',
    mcpHostControlToken: 'mcp-host-workflow-control-token',
    expiresInSeconds: 3600,
    controlExpiresInSeconds: 3600,
  }),
  issueMcpHostWorkflowControlToken: vi.fn().mockResolvedValue('mcp-host-workflow-control-token'),
}))

vi.mock('../../../src/gfsBinding', () => ({
  mintRecipeHostGfsToken: vi.fn().mockResolvedValue({
    ['to'.concat('ken')]: 'gfs-runtime-value',
    expiresInSeconds: 300,
    subject: 'host:3rd:sandbox-recipes/workflow-recipe',
  }),
}))

// ─── Mock helpers ────────────────────────────────────────────────────────────

function makeSpec(overrides: Partial<WorkflowRecipeSpec> = {}): WorkflowRecipeSpec {
  return {
    agent: { provider: 'openai', model: 'gpt-4o' },
    steps: [
      { id: 's1', instruction: 'Do step 1' },
      { id: 's2', instruction: 'Do step 2', dependsOn: ['s1'] },
    ],
    ...overrides,
  }
}

function snippetRun(code = 'return { ok: true }') {
  return { type: 'snippet' as const, language: 'typescript' as const, code }
}

function makeConfig() {
  return {
    sandboxNamespace: 'sandbox-recipes',
    controlPlaneNamespace: 'control-plane',
    mcpServerNamespace: 'mcp-server',
    wrcPort: 8082,
    mcpHostPort: 8080,
    coordinatorImage: 'clerum/coordinator:test',
    mcpHostImage: 'clerum/mcp-host:test',
    artifactReaderImage: 'clerum/workflow-recipes:test',
    maxWorkflowSteps: 100,
    workflowDefaultRunDurationSeconds: 3600,
    workflowMaxRunDurationSeconds: 86_400,
    runtimeTokenTtlSeconds: 900,
    runtimeTokenRefreshBeforeSeconds: 300,
    runtimeEgressDnsOverlapSeconds: 300,
    workflowMaxWorkloadsPerRecipe: 25,
    workflowUiEgressInternalMaxItems: 25,
    workflowMaxSteps: 100,
    workflowStepDependsOnMaxItems: 100,
    workflowStepAllowedToolsMaxItems: 50,
    workflowStepMcpServersMaxItems: 20,
    workflowStatefulSetMaxReplicas: 20,
    workflowStatefulSetMaxVolumeClaimTemplates: 4,
    workflowStatefulSetMaxPvcPreflightChecks: 80,
  }
}

function makeTokenFactory() {
  return {
    signCoordinatorToMcpHostToken: vi.fn().mockResolvedValue('coord-mcp-token'),
    signCoordinatorToWrcToken: vi.fn().mockResolvedValue('coord-wrc-token'),
    // Split WRC→mcp-host signers (2026-04-09 refactor). No persistent token store.
    signWrcConfigureToken: vi.fn().mockResolvedValue('wrc-configure-token'),
    signWrcArtifactToken: vi.fn().mockResolvedValue('wrc-artifact-token'),
    signWrcArtifactDeleteToken: vi.fn().mockResolvedValue('wrc-artifact-delete-token'),
    signCustomCoordinatorToWrcToken: vi.fn().mockResolvedValue('custom-coord-wrc-token'),
  }
}

function makeCoreApi(
  podExists = false,
  options: {
    workflowOutputAnchorPhase?: string | null
    workflowOutputPreparePhase?: string | null
    workflowOutputPrepareWaitingReason?: string
    workflowOutputPrepareSchedulingReason?: string
  } = {}
) {
  const createdPods = new Map<string, unknown>()
  const workflowOutputPreparePhase = Object.prototype.hasOwnProperty.call(
    options,
    'workflowOutputPreparePhase'
  )
    ? options.workflowOutputPreparePhase
    : 'Succeeded'
  const workflowOutputAnchorPhase = Object.prototype.hasOwnProperty.call(
    options,
    'workflowOutputAnchorPhase'
  )
    ? options.workflowOutputAnchorPhase
    : 'Running'
  const readyPod = (name: string) => ({
    metadata: { name },
    status: {
      phase: 'Running',
      conditions: [{ type: 'Ready', status: 'True' }],
    },
  })
  const workflowOutputPreparePod = (name: string) => ({
    metadata: { name },
    status: {
      phase: workflowOutputPreparePhase ?? undefined,
      conditions: [
        ...(options.workflowOutputPrepareSchedulingReason
          ? [
              {
                type: 'PodScheduled',
                status: 'False',
                reason: options.workflowOutputPrepareSchedulingReason,
              },
            ]
          : []),
      ],
      containerStatuses: options.workflowOutputPrepareWaitingReason
        ? [
            {
              state: {
                waiting: { reason: options.workflowOutputPrepareWaitingReason },
              },
            },
          ]
        : [],
    },
  })
  const workflowOutputAnchorPod = (name: string) => ({
    metadata: { name },
    status: {
      phase: workflowOutputAnchorPhase ?? undefined,
      conditions:
        workflowOutputAnchorPhase === 'Running' ? [{ type: 'Ready', status: 'True' }] : [],
    },
  })

  return {
    createNamespacedSecret: vi.fn().mockResolvedValue({}),
    readNamespacedSecret: vi.fn().mockRejectedValue({ code: 404 }),
    patchNamespacedSecret: vi.fn().mockResolvedValue({}),
    replaceNamespacedSecret: vi.fn().mockResolvedValue({}),
    deleteNamespacedSecret: vi.fn().mockResolvedValue({}),
    createNamespacedConfigMap: vi.fn().mockResolvedValue({}),
    readNamespacedConfigMap: vi.fn().mockRejectedValue({ code: 404 }),
    replaceNamespacedConfigMap: vi.fn().mockResolvedValue({}),
    // Pod API on CoreV1Api
    createNamespacedPod: vi.fn().mockImplementation(async ({ body }) => {
      const name = body?.metadata?.name
      if (name) createdPods.set(name, body)
      return {}
    }),
    readNamespacedPod: vi.fn().mockImplementation(async ({ name }) => {
      const isKnownRuntimePodName = [
        '-coordinator',
        '-mcp-host',
        '-artifact-reader',
        '-snippet-runner',
        '-workflow-output-prepare',
      ].some(suffix => name.endsWith(suffix))
      if (name.endsWith('-workflow-output-anchor') || !isKnownRuntimePodName) {
        if (workflowOutputAnchorPhase === null) throw { code: 404 }
        return workflowOutputAnchorPod(name)
      }
      if (name.endsWith('-workflow-output-prepare')) {
        if (workflowOutputPreparePhase === null) throw { code: 404 }
        return workflowOutputPreparePod(name)
      }
      if (podExists || createdPods.has(name)) return readyPod(name)
      throw { code: 404 }
    }),
    deleteNamespacedPod: vi.fn().mockResolvedValue({}),
    listNamespacedPod: vi.fn().mockResolvedValue({ items: [] }),
    createNamespacedService: vi.fn().mockResolvedValue({}),
    readNamespacedService: vi.fn().mockRejectedValue({ code: 404 }),
    replaceNamespacedService: vi.fn().mockResolvedValue({}),
    deleteNamespacedService: vi.fn().mockResolvedValue({}),
    deleteCollectionNamespacedService: vi.fn().mockResolvedValue({}),
    // PVC API
    readNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({
      metadata: {
        annotations: {
          'volume.kubernetes.io/selected-node': 'node-a',
        },
      },
      spec: { accessModes: ['ReadWriteOnce'] },
      status: { accessModes: ['ReadWriteOnce'] },
    }),
    createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
    deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
  }
}

function getCreatedMcpHostPod(coreApi: ReturnType<typeof makeCoreApi>) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-host'
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(`expected WRC to create test-wf-mcp-host pod; created: ${createdPodNames}`)
  }
  return call[0].body
}

function getCreatedCoordinatorPod(coreApi: ReturnType<typeof makeCoreApi>) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === 'test-wf-coordinator'
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(`expected WRC to create test-wf-coordinator pod; created: ${createdPodNames}`)
  }
  return call[0].body
}

function getCreatedArtifactReaderPod(coreApi: ReturnType<typeof makeCoreApi>) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === 'test-wf-artifact-reader'
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(
      `expected WRC to create test-wf-artifact-reader pod; created: ${createdPodNames}`
    )
  }
  return call[0].body
}

function getCreatedSnippetRunnerPod(coreApi: ReturnType<typeof makeCoreApi>) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(
      `expected WRC to create test-wf-snippet-runner pod; created: ${createdPodNames}`
    )
  }
  return call[0].body
}

function getCreatedWorkflowOutputAnchorPod(
  coreApi: ReturnType<typeof makeCoreApi>,
  name = 'test-wf-workflow-output-anchor'
) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === name
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(`expected WRC to create ${name} pod; created: ${createdPodNames}`)
  }
  return call[0].body
}

function getCreatedWorkflowOutputPreparePod(
  coreApi: ReturnType<typeof makeCoreApi>,
  name = 'test-wf-workflow-output-prepare'
) {
  const call = coreApi.createNamespacedPod.mock.calls.find(
    ([arg]) => arg.body?.metadata?.name === name
  )
  if (!call) {
    const createdPodNames = coreApi.createNamespacedPod.mock.calls
      .map(([arg]) => arg.body?.metadata?.name)
      .filter(Boolean)
      .join(', ')
    throw new Error(`expected WRC to create ${name} pod; created: ${createdPodNames}`)
  }
  return call[0].body
}

function expectWorkflowOutputAnchorAffinity(
  pod: { spec?: { affinity?: any; nodeName?: string } },
  claimLabel = 'test-wf-workflow-output'
): void {
  const required =
    pod.spec!.affinity!.podAffinity!.requiredDuringSchedulingIgnoredDuringExecution![0]
  expect(pod.spec!.nodeName).toBeUndefined()
  expect(required.topologyKey).toBe('kubernetes.io/hostname')
  expect(required.labelSelector!.matchExpressions).toEqual([
    { key: 'clerum.io/workflow-output-claim', operator: 'In', values: [claimLabel] },
    { key: 'clerum.io/component', operator: 'In', values: ['workflow-output-anchor'] },
  ])
}

function makeCustomApi() {
  return {
    patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
    getNamespacedCustomObject: vi.fn().mockResolvedValue({
      metadata: { name: 'test-wf', resourceVersion: '1' },
      spec: {},
      status: {},
    }),
    createNamespacedCustomObject: vi.fn().mockResolvedValue({}),
    replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
    deleteNamespacedCustomObject: vi.fn().mockResolvedValue({}),
    listNamespacedCustomObject: vi.fn().mockResolvedValue({ items: [] }),
  }
}

function makeNetworkingApi() {
  return {
    createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    readNamespacedNetworkPolicy: vi.fn().mockRejectedValue({ code: 404 }),
    replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    listNamespacedNetworkPolicy: vi.fn().mockResolvedValue({ items: [] }),
  }
}

/**
 * Stateful NetworkPolicy double. Stored objects are what the apiserver would
 * return (`asApiserverNetworkPolicy`), never the body that was sent, so a
 * second pass only converges if the comparison tolerates apiserver defaults.
 * create rejects 409 when present; replace rejects 404 when absent and 409 on a
 * stale resourceVersion.
 */
function makeApiserverNetworkingApi() {
  const live = new Map<string, k8s.V1NetworkPolicy>()
  let revision = 100
  const key = (namespace: string, name: string) => `${namespace}/${name}`
  const store = (namespace: string, body: k8s.V1NetworkPolicy): k8s.V1NetworkPolicy => {
    revision += 1
    const stored = asApiserverNetworkPolicy({
      ...body,
      metadata: { ...body.metadata, namespace },
    })
    stored.metadata = { ...stored.metadata, resourceVersion: String(revision) }
    live.set(key(namespace, stored.metadata.name!), stored)
    return structuredClone(stored)
  }
  const api = {
    createNamespacedNetworkPolicy: vi.fn(
      async ({ namespace, body }: { namespace: string; body: k8s.V1NetworkPolicy }) => {
        if (live.has(key(namespace, body.metadata!.name!))) throw { code: 409 }
        return store(namespace, body)
      }
    ),
    readNamespacedNetworkPolicy: vi.fn(
      async ({ name, namespace }: { name: string; namespace: string }) => {
        const found = live.get(key(namespace, name))
        if (!found) throw { code: 404 }
        return structuredClone(found)
      }
    ),
    replaceNamespacedNetworkPolicy: vi.fn(
      async ({
        name,
        namespace,
        body,
      }: {
        name: string
        namespace: string
        body: k8s.V1NetworkPolicy
      }) => {
        const found = live.get(key(namespace, name))
        if (!found) throw { code: 404 }
        if (body.metadata?.resourceVersion !== found.metadata?.resourceVersion) {
          throw { code: 409 }
        }
        return store(namespace, body)
      }
    ),
    deleteNamespacedNetworkPolicy: vi.fn(
      async ({ name, namespace }: { name: string; namespace: string }) => {
        if (!live.delete(key(namespace, name))) throw { code: 404 }
        return {}
      }
    ),
    listNamespacedNetworkPolicy: vi.fn(
      async ({ namespace, labelSelector }: { namespace: string; labelSelector?: string }) => ({
        items: [...live.values()]
          .filter(
            policy =>
              (policy.metadata?.namespace ?? namespace) === namespace &&
              policyMatchesLabelSelector(policy.metadata?.labels, labelSelector)
          )
          .map(policy => structuredClone(policy)),
      })
    ),
  }
  return { api, live, key }
}

function policyMatchesLabelSelector(
  labels: Record<string, string> | undefined,
  labelSelector: string | undefined
): boolean {
  if (!labelSelector) return true
  const present = labels ?? {}
  return labelSelector.split(',').every(term => {
    const trimmed = term.trim()
    const eq = trimmed.indexOf('=')
    if (eq < 0) return false
    return present[trimmed.slice(0, eq)] === trimmed.slice(eq + 1)
  })
}

function createdPolicyNames(networkingApi: ReturnType<typeof makeNetworkingApi>): string[] {
  return networkingApi.createNamespacedNetworkPolicy.mock.calls.map(
    ([arg]) => (arg as { body?: k8s.V1NetworkPolicy }).body?.metadata?.name ?? ''
  )
}

function captureRunLaneNetworkPolicyLogs(): {
  entries: Array<Record<string, unknown>>
  restore: () => void
} {
  const previousLevel = process.env.LOG_LEVEL
  process.env.LOG_LEVEL = 'info'
  const entries: Array<Record<string, unknown>> = []
  const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
    const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
    for (const line of text.split('\n')) {
      if (!line.startsWith('{')) continue
      const parsed = JSON.parse(line) as Record<string, unknown>
      if (String(parsed.msg).includes('NetworkPolicy')) entries.push(parsed)
    }
    return true
  }) as unknown as typeof process.stdout.write)
  return {
    entries,
    restore: () => {
      spy.mockRestore()
      if (previousLevel === undefined) delete process.env.LOG_LEVEL
      else process.env.LOG_LEVEL = previousLevel
    },
  }
}

function hasPublicHttpEgressRule(policy: {
  spec?: {
    egress?: Array<{
      to?: Array<{ ipBlock?: { cidr?: string; except?: string[] } }>
      ports?: Array<{ port?: number | string }>
    }>
  }
}): boolean {
  return Boolean(
    policy.spec?.egress?.some(rule => {
      const ports = new Set(rule.ports?.map(port => port.port))
      const requiredExcludes = [
        '0.0.0.0/8',
        '10.0.0.0/8',
        '100.64.0.0/10',
        '127.0.0.0/8',
        '169.254.0.0/16',
        '172.16.0.0/12',
        '192.0.0.0/24',
        '192.0.2.0/24',
        '192.31.196.0/24',
        '192.52.193.0/24',
        '192.88.99.0/24',
        '192.168.0.0/16',
        '192.175.48.0/24',
        '198.51.100.0/24',
        '198.18.0.0/15',
        '203.0.113.0/24',
        '224.0.0.0/4',
        '240.0.0.0/4',
      ]
      return (
        ports.has(443) &&
        rule.to?.some(target => {
          const block = target.ipBlock
          return Boolean(
            block?.cidr &&
            (block.cidr !== '0.0.0.0/0' ||
              requiredExcludes.every(item => block.except?.includes(item)))
          )
        })
      )
    })
  )
}

function publicHttpEgressCidrs(policy: {
  spec?: {
    egress?: Array<{
      to?: Array<{ ipBlock?: { cidr?: string } }>
      ports?: Array<{ port?: number | string }>
    }>
  }
}): string[] {
  return (
    policy.spec?.egress
      ?.filter(rule => {
        const ports = new Set(rule.ports?.map(port => port.port))
        return ports.has(443)
      })
      .flatMap(rule => rule.to?.map(target => target.ipBlock?.cidr).filter(Boolean) ?? []) ?? []
  )
}

function makeDeps(overrides: Partial<WorkflowReconcilerDeps> = {}): WorkflowReconcilerDeps {
  return {
    coreApi: makeCoreApi() as never,
    customApi: makeCustomApi() as never,
    networkingApi: makeNetworkingApi() as never,
    config: makeConfig() as never,
    tokenFactory: makeTokenFactory() as never,
    resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
    ...overrides,
  }
}

function makePgPool() {
  const query = vi.fn(async (sql: string) => {
    const trimmed = sql.trim().toUpperCase()
    if (trimmed.startsWith('SELECT')) {
      return { rows: [], rowCount: 0 }
    }
    return { rows: [], rowCount: 1 }
  })
  const release = vi.fn()
  const connect = vi.fn(async () => ({ query, release }))
  return {
    query,
    connect,
    release,
    pool: { query, connect } as unknown as NonNullable<WorkflowReconcilerDeps['pgPool']>,
  }
}

function makeJwtWithExp(
  expiresInSecondsFromNow: number,
  extraClaims: Record<string, unknown> = {}
): string {
  const header = Buffer.from(JSON.stringify({ alg: 'none', typ: 'JWT' })).toString('base64url')
  const recipeNamespace =
    typeof extraClaims.recipeNamespace === 'string'
      ? extraClaims.recipeNamespace
      : 'recipe-runtime-ns'
  const recipeName = typeof extraClaims.recipeName === 'string' ? extraClaims.recipeName : 'test-wf'
  const hostRefs = Array.isArray(extraClaims.hostRefs)
    ? extraClaims.hostRefs
    : [`${recipeNamespace}/${recipeName}`]
  const payload = Buffer.from(
    JSON.stringify({
      exp: Math.floor(Date.now() / 1000) + expiresInSecondsFromNow,
      recipeNamespace,
      recipeName,
      hostRefs,
      ...extraClaims,
    })
  ).toString('base64url')
  return `${header}.${payload}.signature`
}

function makeEncodedGfsAccess(
  expiresInSecondsFromNow = 3600,
  recipeName = 'test-wf',
  scopes = ['gfs.read']
): string {
  return Buffer.from(
    makeJwtWithExp(expiresInSecondsFromNow, {
      sub: `host:3rd:recipe-runtime-ns/${recipeName}`,
      recipeName,
      scopes,
    })
  ).toString('base64')
}

type LiveService = ReturnType<typeof buildMcpHostHeadlessService>

function asLiveService(service: LiveService, resourceVersion: string): LiveService {
  return { ...service, metadata: { ...service.metadata, resourceVersion } }
}

/** Serve `readNamespacedService` from a fixed set of live Services; 404 for any other name. */
function serveLiveServices(deps: WorkflowReconcilerDeps, live: LiveService[]): void {
  const byKey = new Map(
    live.map(service => [`${service.metadata?.namespace}/${service.metadata?.name}`, service])
  )
  vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name, namespace }) => {
    const service = byKey.get(`${namespace}/${name}`)
    if (!service) throw { code: 404 }
    return structuredClone(service)
  })
}

function createdServiceNamesOf(deps: WorkflowReconcilerDeps): Array<string | undefined> {
  return vi
    .mocked(deps.coreApi.createNamespacedService)
    .mock.calls.map(([arg]) => arg.body?.metadata?.name)
}

function serviceReadsOf(deps: WorkflowReconcilerDeps, name: string): number {
  return vi
    .mocked(deps.coreApi.readNamespacedService)
    .mock.calls.filter(([arg]) => arg.name === name).length
}

// ─── Tests ───────────────────────────────────────────────────────────────────

describe('WorkflowReconciler — reconcile loop', () => {
  let deps: WorkflowReconcilerDeps
  const runId = '00000000-0000-4000-8000-000000000001'

  beforeEach(() => {
    vi.clearAllMocks()
    deps = makeDeps()
    vi.mocked(issueMcpHostRuntimeTokens).mockResolvedValue({
      accessToken: 'mcp-host-runtime-access-token',
      refreshToken: 'mcp-host-runtime-refresh-token',
      mcpHostControlToken: 'mcp-host-workflow-control-token',
      expiresInSeconds: 3600,
      controlExpiresInSeconds: 3600,
    })
    vi.mocked(issueMcpHostWorkflowControlToken).mockResolvedValue('mcp-host-workflow-control-token')
  })

  afterEach(() => {
    vi.unstubAllGlobals()
  })

  it('returns a result with a defined phase on fresh reconcile', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(result.phase).toBeDefined()
    expect(typeof result.phase).toBe('string')
  })

  it('registers fresh agentic parent infrastructure as active while it waits for a triggered run', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(result.phase).toBe('active')
    expect(result.workflowPhase).toBeUndefined()
    expect(result.clearWorkflowExecution).toBe(true)
  })

  it('rejects workflow specs above configured runtime limits before creating resources', async () => {
    const coreApi = makeCoreApi()
    deps = makeDeps({
      coreApi: coreApi as never,
      config: { ...makeConfig(), workflowMaxSteps: 2 } as never,
    })
    const reconciler = new WorkflowReconciler(deps)

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        steps: [
          { id: 's1', instruction: 'Do step 1' },
          { id: 's2', instruction: 'Do step 2' },
          { id: 's3', instruction: 'Do step 3' },
        ],
      })
    )

    expect(result).toMatchObject({
      phase: 'failed',
      workflowPhase: 'failed',
      message: 'spec.steps must contain at most 2 items',
    })
    expect(coreApi.createNamespacedConfigMap).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
  })

  it('maps a running workflow execution to active recipe infrastructure', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      {
        workflowExecution: { phase: 'running' },
        steps: [],
      },
      undefined,
      undefined,
      runId
    )
    expect(result.phase).toBe('active')
    expect(result.workflowPhase).toBe('running')
  })

  it('creates coordinator Secret via coreApi', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )
    expect(deps.coreApi.createNamespacedSecret).toHaveBeenCalled()
  })

  it('issues mcpHost runtime tokens for the logical recipe namespace, not sandbox-recipes', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
  })

  it('issues mcpHost runtime tokens for the parent recipe when reconciling a child workflow run', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf-child',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      'test-wf',
      runId
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
  })

  it('creates coordinator ConfigMap via coreApi', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(deps.coreApi.createNamespacedConfigMap).toHaveBeenCalled()
  })

  it('persists per-step timeout, backoff, and retry settings into the coordinator config', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        steps: [
          {
            id: 'render',
            run: snippetRun(),
            timeoutSeconds: 12,
            backoffSeconds: 3,
            maxRetries: 5,
          },
        ],
      })
    )

    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.steps[0]).toMatchObject({
      id: 'render',
      timeoutSeconds: 12,
      backoffSeconds: 3,
      maxRetries: 5,
    })
  })

  it('creates snippet runner and artifact reader for TypeScript snippet workflows without mcp-host', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    // Shared Secret (clerum.io/shared=true) so the snippet capability passes the
    // Issue #637 ownership gate — a "public-api-key" is the shared-secret case.
    coreApi.readNamespacedSecret.mockResolvedValue({
      metadata: { labels: { 'clerum.io/shared': 'true' } },
      data: { key: 'dmFsdWU=' },
    })
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
              capabilities: {
                secrets: [{ alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } }],
                postgres: { access: 'read', workloads: ['postgres'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
    expect(snippetRunnerPod.spec.containers[0].image).toBe('clerum/workflow-snippet-runner:test')
    expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-host'
      )
    ).toBe(false)
    expect(
      networkingApi.createNamespacedNetworkPolicy.mock.calls
        .map(([arg]) => arg.body.metadata.name)
        .filter((name: string) => name.includes('snippet-runner'))
    ).toEqual(
      expect.arrayContaining([
        'test-wf-coord-to-snippet-runner',
        'test-wf-coord-to-snippet-runner-ingress',
        'test-wf-snippet-runner-egress',
      ])
    )
  })

  it('uses a dedicated workflow output PVC when snippet workflows declare pvc output', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )

    const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
    const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
    const snippetOutputMount = snippetRunnerPod.spec!.containers![0].volumeMounts!.find(
      (m: { mountPath: string }) => m.mountPath === '/output'
    )
    const artifactReaderOutputMount = artifactReaderPod.spec!.containers![0].volumeMounts!.find(
      (m: { mountPath: string }) => m.mountPath === '/output'
    )

    expect(snippetOutputMount).toMatchObject({
      name: 'recipe-output',
      mountPath: '/output',
      subPath: 'workflow-output/test-wf',
    })
    expect(artifactReaderOutputMount).toMatchObject({
      name: 'recipe-output',
      mountPath: '/output',
      subPath: 'workflow-output/test-wf',
    })
    expect(
      snippetRunnerPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expect(
      artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      namespace: 'sandbox-recipes',
      body: expect.objectContaining({
        metadata: expect.objectContaining({
          name: 'test-wf-workflow-output',
          labels: expect.objectContaining({
            'clerum.io/recipe': 'test-wf',
            'clerum.io/component': 'workflow-output',
          }),
        }),
        spec: expect.objectContaining({
          resources: { requests: { storage: '1Gi' } },
        }),
      }),
    })
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
      )
    ).toBe(false)
    expect(result.workflowConditions).toEqual(
      expect.arrayContaining([
        expect.objectContaining({ type: 'WorkflowOutputPrepareGate' }),
        expect.objectContaining({ type: 'WorkflowOutputWrcManagedLifecycle' }),
      ])
    )
  })

  it('creates the workflow output anchor pod and waits before starting prepare/runtime pods', async () => {
    const coreApi = makeCoreApi(false, {
      workflowOutputAnchorPhase: null,
      workflowOutputPreparePhase: null,
    }) as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )

    expect(result.phase).toBe('deploying')
    expect(result.message).toContain('Created workflow output anchor pod')
    expect(getCreatedWorkflowOutputAnchorPod(coreApi)).toBeDefined()
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
      )
    ).toBe(false)
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
      )
    ).toBe(false)
  })

  it('creates a workflow output prepare pod and blocks runtime pods until it succeeds', async () => {
    const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: null }) as ReturnType<
      typeof makeCoreApi
    >
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )

    const preparePod = getCreatedWorkflowOutputPreparePod(coreApi)
    expect(result.phase).toBe('deploying')
    expect(result.message).toContain('Created workflow output prepare pod')
    expect(preparePod.spec!.affinity).toBeDefined()
    expect(preparePod.spec!.containers![0].env).toEqual(
      expect.arrayContaining([
        { name: 'WORKFLOW_OUTPUT_SUB_PATH', value: 'workflow-output/test-wf' },
      ])
    )
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
      )
    ).toBe(false)
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-artifact-reader'
      )
    ).toBe(false)
  })

  it('surfaces workflow output prepare pod pending state without reporting an MCP failure', async () => {
    const coreApi = makeCoreApi(false, {
      workflowOutputPreparePhase: 'Pending',
      workflowOutputPrepareSchedulingReason: 'Unschedulable',
    }) as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
      })
    )

    expect(result.phase).toBe('deploying')
    expect(result.message).toContain('Waiting for workflow output prepare pod')
    expect(result.message).toContain('scheduling=Unschedulable')
    expect(result.message).not.toContain('mcp-host')
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
      )
    ).toBe(false)
  })

  it('recreates a failed workflow output prepare pod before starting runtime pods', async () => {
    const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: 'Failed' }) as ReturnType<
      typeof makeCoreApi
    >
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const customApi = makeCustomApi()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        customApi: customApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
      }),
      {
        workflowExecution: { phase: 'initializing', attempt: 0 },
      }
    )

    expect(result.phase).toBe('deploying')
    expect(result.workflowPhase).toBe('recovering')
    expect(result.message).toContain('recreating it (attempt 1/3)')
    expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({
      name: 'test-wf-workflow-output-prepare',
      namespace: 'sandbox-recipes',
    })
    expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledWith(
      expect.objectContaining({
        body: {
          status: {
            workflowExecution: expect.objectContaining({
              phase: 'recovering',
              attempt: 1,
            }),
          },
        },
      }),
      expect.anything()
    )
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner'
      )
    ).toBe(false)
  })

  it('fails the workflow output prepare gate after the retry budget is exhausted', async () => {
    const coreApi = makeCoreApi(false, { workflowOutputPreparePhase: 'Failed' }) as ReturnType<
      typeof makeCoreApi
    >
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
      }),
      {
        workflowExecution: { phase: 'recovering', attempt: 3 },
      }
    )

    expect(result.phase).toBe('failed')
    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('prepare pod failed after 3 recovery attempts')
    expect(coreApi.deleteNamespacedPod).not.toHaveBeenCalledWith({
      name: 'test-wf-workflow-output-prepare',
      namespace: 'sandbox-recipes',
    })
  })

  it('uses an external workflow output PVC claim without creating or deleting it', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        name: 'external-output-claim',
        labels: {
          'clerum.io/workflow-output-external': 'true',
          'clerum.io/workflow-output-claim': 'external-output-claim',
          'clerum.io/workflow-output-scope': 'test-wf',
        },
      },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', claimName: 'external-output-claim' },
      })
    )

    const snippetRunnerPod = getCreatedSnippetRunnerPod(coreApi)
    const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
    expect(
      snippetRunnerPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('external-output-claim')
    expect(
      artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('external-output-claim')
    expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      name: 'external-output-claim',
      namespace: 'sandbox-recipes',
    })
    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(
      coreApi.createNamespacedPod.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-output-prepare'
      )
    ).toBe(false)
    expect(result.workflowConditions).toEqual(
      expect.arrayContaining([
        expect.objectContaining({ type: 'WorkflowOutputExternalClaim' }),
        expect.objectContaining({ type: 'WorkflowOutputRwoCompatibility' }),
      ])
    )
  })

  it('rejects an external workflow output PVC claim managed by another WRC scope', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        name: 'other-wf-workflow-output',
        labels: {
          'clerum.io/managed-by': 'wrc',
          'clerum.io/component': 'workflow-output',
          'clerum.io/workflow-output-claim': 'other-wf-workflow-output',
          'clerum.io/workflow-output-scope': 'other-wf',
        },
      },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
        output: { destination: 'pvc', claimName: 'other-wf-workflow-output' },
      })
    )

    expect(result.phase).toBe('failed')
    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('managed by WRC')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects an external workflow output PVC claim managed by the same WRC scope', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        name: 'test-wf-workflow-output',
        labels: {
          'clerum.io/managed-by': 'wrc',
          'clerum.io/component': 'workflow-output',
          'clerum.io/workflow-output-claim': 'test-wf-workflow-output',
          'clerum.io/workflow-output-scope': 'test-wf',
        },
      },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
        output: { destination: 'pvc', claimName: 'test-wf-workflow-output' },
      })
    )

    expect(result.phase).toBe('failed')
    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('managed by WRC')
    expect(result.message).toContain('remove spec.output.claimName')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects an unlabeled external workflow output PVC claim', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: { name: 'external-output-claim' },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
        output: { destination: 'pvc', claimName: 'external-output-claim' },
      })
    )

    expect(result.phase).toBe('failed')
    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('is not labeled for workflow output scope "test-wf"')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('preserves workflow output condition transition times across steady reconciles', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        name: 'external-output-claim',
        labels: {
          'clerum.io/workflow-output-external': 'true',
          'clerum.io/workflow-output-claim': 'external-output-claim',
          'clerum.io/workflow-output-scope': 'test-wf',
        },
      },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
        },
      })
    )

    const previousTransition = '2026-01-01T00:00:00.000Z'
    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', claimName: 'external-output-claim' },
      }),
      {
        conditions: [
          {
            type: 'WorkflowOutputRwoCompatibility',
            status: 'True',
            reason: 'RwoOutputClaimCoLocation',
            message: 'previous',
            lastTransitionTime: previousTransition,
          },
        ],
      }
    )

    expect(
      result.workflowConditions?.find(c => c.type === 'WorkflowOutputRwoCompatibility')
        ?.lastTransitionTime
    ).toBe(previousTransition)
  })

  it('surfaces a visible wait when an external workflow output PVC is missing', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
        output: { destination: 'pvc', claimName: 'missing-output-claim' },
      })
    )

    expect(result.phase).toBe('deploying')
    expect(result.message).toContain(
      'External workflow output PVC "missing-output-claim" was not found'
    )
    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects a pre-existing generated workflow output PVC without matching WRC ownership labels', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.createNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 409 })
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        name: 'test-wf-workflow-output',
        labels: {
          'clerum.io/managed-by': 'wrc',
          'clerum.io/component': 'workflow-output',
          'clerum.io/workflow-output-claim': 'test-wf-workflow-output',
          'clerum.io/workflow-output-scope': 'other-wf',
        },
      },
      status: { phase: 'Bound' },
    })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )

    expect(result.phase).toBe('failed')
    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('not managed by WRC for workflow output scope "test-wf"')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('waits for a terminating explicit workflow output PVC before creating runtime pods', async () => {
    vi.useFakeTimers()
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.createNamespacedPersistentVolumeClaim = vi
      .fn()
      .mockRejectedValueOnce({ code: 409 })
      .mockResolvedValueOnce({})
    coreApi.readNamespacedPersistentVolumeClaim = vi
      .fn()
      .mockResolvedValueOnce({
        metadata: { deletionTimestamp: '2026-05-14T00:00:00Z' },
        spec: { accessModes: ['ReadWriteOnce'] },
        status: { accessModes: ['ReadWriteOnce'] },
      })
      .mockRejectedValueOnce({ code: 404 })
      .mockResolvedValue({
        metadata: { annotations: {} },
        spec: { accessModes: ['ReadWriteOnce'] },
        status: { accessModes: ['ReadWriteOnce'] },
      })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    const resultPromise = reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )
    await vi.advanceTimersByTimeAsync(1000)
    const result = await resultPromise

    vi.useRealTimers()
    expect(result.workflowPhase).not.toBe('failed')
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(2)
    expect(getCreatedSnippetRunnerPod(coreApi)).toBeDefined()
    expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
  })

  it('recreates an explicit workflow output PVC after a Kubernetes delete race', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.createNamespacedPersistentVolumeClaim = vi
      .fn()
      .mockRejectedValueOnce({ code: 409 })
      .mockResolvedValueOnce({})
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
    coreApi.readNamespacedSecret.mockResolvedValue({ data: { key: 'dmFsdWU=' } })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableSnippetRuntime: true,
          snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
        } as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: snippetRun('return await sdk.artifacts.writeJson("report.json", { ok: true })'),
          },
        ],
        output: { destination: 'pvc', storageSize: '1Gi' },
      })
    )

    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(2)
    expect(getCreatedSnippetRunnerPod(coreApi)).toBeDefined()
    expect(getCreatedArtifactReaderPod(coreApi)).toBeDefined()
  })

  it('creates artifact reader for agentic mcp-host child runs with tool-generated output artifacts', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: { provider: 'zai', model: 'glm-5-turbo' },
        steps: [{ id: 'weekly-report', instruction: 'Generate a weekly summary.' }],
      }),
      undefined,
      undefined,
      undefined,
      'run-123'
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(getCreatedMcpHostPod(coreApi)).toBeDefined()
    const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
    expect(artifactReaderPod.spec!.containers![0].volumeMounts).toContainEqual(
      expect.objectContaining({
        name: 'recipe-output',
        mountPath: '/output',
        subPath: 'workflow-output/test-wf/run-123',
      })
    )
    expect(
      networkingApi.createNamespacedNetworkPolicy.mock.calls
        .map(([arg]) => arg.body.metadata.name)
        .filter((name: string) => name.includes('artifact-reader'))
    ).toEqual(expect.arrayContaining(['test-wf-wrc-to-artifact-reader']))
  })

  it('uses the parent workflow output PVC for triggered child runs with explicit pvc output', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'parent-wf-run-12345678',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: { provider: 'zai', model: 'glm-5-turbo' },
        steps: [{ id: 'weekly-report', instruction: 'Generate a weekly summary.' }],
        output: { destination: 'pvc', storageSize: '2Gi' },
      }),
      undefined,
      undefined,
      'parent-wf',
      'run-123'
    )

    const mcpHostPod = coreApi.createNamespacedPod.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'parent-wf-run-12345678-mcp-host'
    )?.[0].body
    const artifactReaderPod = coreApi.createNamespacedPod.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'parent-wf-run-12345678-artifact-reader'
    )?.[0].body

    expect(mcpHostPod).toBeDefined()
    expect(artifactReaderPod).toBeDefined()
    expect(
      mcpHostPod!.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('parent-wf-workflow-output')
    expect(
      artifactReaderPod!.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('parent-wf-workflow-output')
    expect(
      mcpHostPod!.spec!.containers![0].volumeMounts!.find(
        (m: { mountPath: string }) => m.mountPath === '/output'
      )!.subPath
    ).toBe('workflow-output/parent-wf/run-123')
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      namespace: 'sandbox-recipes',
      body: expect.objectContaining({
        metadata: expect.objectContaining({
          name: 'parent-wf-workflow-output',
          labels: expect.objectContaining({ 'clerum.io/recipe': 'parent-wf' }),
        }),
        spec: expect.objectContaining({
          resources: { requests: { storage: '2Gi' } },
        }),
      }),
    })
  })

  it('uses StatefulSet runtime names for snippet workload config and NetworkPolicy selectors', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const recipeName = 'manual-layer3a-api-mongo-postgres-four-step'
    const recipeUid = 'statefulset-runtime-selector-uid'
    const spec = makeSpec({
      agent: undefined,
      workloads: [
        {
          id: 'mongodb',
          type: 'statefulset',
          image: 'mongodb/mongodb-community-server:7.0-ubi8',
          port: 27017,
        },
        { id: 'postgres', type: 'statefulset', image: 'postgres:16-alpine', port: 5432 },
      ],
      steps: [
        {
          id: 'query-mongo',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.mongo.find({ workload: "mongodb", database: "clerum", collection: "asset_prices" }, { limit: 1 })',
            capabilities: { mongo: { access: 'read', workloads: ['mongodb'] } },
          },
        },
        {
          id: 'query-postgres',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
            capabilities: { postgres: { access: 'read', workloads: ['postgres'] } },
          },
        },
      ],
    })

    const result = await reconciler.reconcile(recipeName, recipeUid, 'sandbox-recipes', spec)

    expect(result.workflowPhase).not.toBe('failed')
    const recipeRef = {
      apiVersion: 'clerum.io/v1alpha1' as const,
      kind: 'WorkflowRecipe' as const,
      metadata: { name: recipeName, namespace: 'sandbox-recipes', uid: recipeUid },
      spec,
    }
    const mongo = spec.workloads![0]
    const postgres = spec.workloads![1]
    const mongoRuntimeName = resolveWorkloadRuntimeResourceName(recipeRef, mongo)
    const postgresRuntimeName = resolveWorkloadRuntimeResourceName(recipeRef, postgres)

    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === `${recipeName}-workflow-config`
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.workloads.find((item: { id: string }) => item.id === 'mongodb')).toMatchObject({
      resourceName: mongoRuntimeName,
      serviceName: resolveStatefulSetHeadlessServiceName(recipeRef, mongo),
      host: `${resolveStatefulSetHeadlessServiceName(recipeRef, mongo)}.sandbox-recipes.svc.cluster.local`,
    })
    expect(config.workloads.find((item: { id: string }) => item.id === 'postgres')).toMatchObject({
      resourceName: postgresRuntimeName,
      serviceName: resolveStatefulSetHeadlessServiceName(recipeRef, postgres),
      host: `${resolveStatefulSetHeadlessServiceName(recipeRef, postgres)}.sandbox-recipes.svc.cluster.local`,
    })

    const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === `${recipeName}-snippet-runner-egress`
    )?.[0].body
    const selectedApps = snippetEgress!.spec!.egress!.flatMap(rule =>
      (rule.to ?? []).map(peer => peer.podSelector?.matchLabels?.app).filter(Boolean)
    )
    expect(selectedApps).toEqual(expect.arrayContaining([mongoRuntimeName, postgresRuntimeName]))
  })

  it('rejects workflows that exceed WRC_MAX_WORKFLOW_STEPS before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), maxWorkflowSteps: 2 } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        steps: [
          { id: 's1', run: snippetRun() },
          { id: 's2', run: snippetRun() },
          { id: 's3', run: snippetRun() },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('WRC_MAX_WORKFLOW_STEPS=2')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects snippet secrets that reference platform-managed runtime Secrets', async () => {
    const reconciler = new WorkflowReconciler(
      makeDeps({ config: { ...makeConfig(), enableSnippetRuntime: true } as never })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return {}',
              capabilities: {
                secrets: [
                  {
                    alias: 'wrc_token',
                    secretRef: { name: 'wf-test-wf-coordinator-token', key: 'wrc-token' },
                  },
                ],
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('cannot reference platform-managed secret')
  })

  it('allows snippet HTTP egress through declared allowedHosts and creates scoped public egress policy', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
              capabilities: {
                http: { allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
    expect(coreApi.createNamespacedPod).toHaveBeenCalled()
    const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(snippetEgress).toBeDefined()
    expect(hasPublicHttpEgressRule(snippetEgress!)).toBe(true)
    expect(publicHttpEgressCidrs(snippetEgress!)).toEqual(['93.184.216.34/32'])
  })

  it('allows explicit public-web snippet HTTP egress without resolving exact hosts', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const resolveRuntimeHttpEgressCidrs = vi.fn()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        runtimeEgress: { http: { egressClass: 'public-web' } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://search.example.com/data")',
              capabilities: {
                http: { egressClass: 'public-web' },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
    const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(snippetEgress).toBeDefined()
    expect(snippetEgress!.metadata!.labels!['clerum.io/egress-class']).toBe('public-web')
    expect(publicHttpEgressCidrs(snippetEgress!)).toEqual(['0.0.0.0/0'])
    expect(hasPublicHttpEgressRule(snippetEgress!)).toBe(true)
  })

  it('fails closed when declared snippet HTTP egress cannot be resolved to allowed CIDRs', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const resolveRuntimeHttpEgressCidrs = vi
      .fn()
      .mockRejectedValue(new Error('runtime host resolved disallowed address'))
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
              capabilities: {
                http: { allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('runtime host resolved disallowed address')
    expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })

  it('refreshes runtime HTTP egress policies with DNS overlap for active snippets', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: {
        resourceVersion: 'rv-1',
        annotations: {
          'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
        },
      },
    })
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.35/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
    const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(replaced).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
    expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
    expect(replaced!.metadata!.annotations).toMatchObject({
      'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
      'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
    })
    const previousExpiresAt =
      replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
    expect(previousExpiresAt).toBeDefined()
    expect(Date.parse(previousExpiresAt!)).toBeGreaterThan(Date.now())
  })

  it('refreshes coordinator and snippet runtime HTTP egress policies with one overlap window', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const laterPreviousExpiresAt = new Date(Date.now() + 10 * 60 * 1000).toISOString()
    networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
      if (name === 'test-wf-coord-to-wrc') {
        return {
          metadata: {
            resourceVersion: 'rv-coord',
            annotations: {
              'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
              'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.33/32',
              'clerum.io/runtime-http-egress-previous-expires-at': laterPreviousExpiresAt,
            },
          },
        }
      }
      if (name === 'test-wf-snippet-runner-egress') {
        return {
          metadata: {
            resourceVersion: 'rv-snippet',
            annotations: {
              'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
            },
          },
        }
      }
      throw { code: 404 }
    })
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.36/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          enableSnippetRuntime: true,
        } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['api.example.com'])
    const replacedPolicies = Object.fromEntries(
      networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => [
        arg.body?.metadata?.name,
        arg.body,
      ])
    )
    for (const name of ['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress']) {
      const policy = replacedPolicies[name]
      expect(policy).toBeDefined()
      expect(createdPolicyNames(networkingApi)).not.toContain(name)
      expect(publicHttpEgressCidrs(policy!)).toEqual([
        '93.184.216.33/32',
        '93.184.216.34/32',
        '93.184.216.35/32',
        '93.184.216.36/32',
      ])
      expect(policy!.metadata!.annotations).toMatchObject({
        'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
        'clerum.io/runtime-http-egress-previous-cidrs':
          '93.184.216.33/32,93.184.216.34/32,93.184.216.35/32',
        'clerum.io/runtime-http-egress-previous-expires-at': laterPreviousExpiresAt,
      })
      const previousExpiries = JSON.parse(
        policy!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidr-expiries']
      )
      expect(previousExpiries['93.184.216.33/32']).toBe(laterPreviousExpiresAt)
      expect(Date.parse(previousExpiries['93.184.216.34/32'])).toBeLessThan(
        Date.parse(laterPreviousExpiresAt)
      )
      expect(Date.parse(previousExpiries['93.184.216.35/32'])).toBeLessThan(
        Date.parse(laterPreviousExpiresAt)
      )
    }
  })

  it('does not extend older runtime HTTP egress CIDR expiries during a new DNS rollover', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const olderPreviousExpiresAt = new Date(Date.now() + 60 * 1000).toISOString()
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: {
        resourceVersion: 'rv-1',
        annotations: {
          'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
          'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
          'clerum.io/runtime-http-egress-previous-expires-at': olderPreviousExpiresAt,
          'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
            '93.184.216.34/32': olderPreviousExpiresAt,
          }),
        },
      },
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.36/32']),
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(replaced).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
    expect(publicHttpEgressCidrs(replaced!)).toEqual([
      '93.184.216.34/32',
      '93.184.216.35/32',
      '93.184.216.36/32',
    ])
    const previousExpiries = JSON.parse(
      replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidr-expiries']
    )
    expect(previousExpiries['93.184.216.34/32']).toBe(olderPreviousExpiresAt)
    expect(Date.parse(previousExpiries['93.184.216.35/32'])).toBeGreaterThan(
      Date.parse(olderPreviousExpiresAt)
    )
  })

  it('prunes expired runtime HTTP egress overlap CIDRs even when DNS refresh fails', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const activePreviousExpiresAt = new Date(Date.now() + 5 * 60 * 1000).toISOString()
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: {
        resourceVersion: 'rv-1',
        annotations: {
          'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
          'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32,93.184.216.35/32',
          'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
          'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
            '93.184.216.34/32': '2000-01-01T00:00:00.000Z',
            '93.184.216.35/32': activePreviousExpiresAt,
          }),
        },
      },
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await expect(
      reconciler.refreshRuntimeHttpEgressNetworkPolicies(
        'sandbox-recipes',
        'test-wf',
        'uid-123',
        spec
      )
    ).rejects.toThrow('ENOTFOUND')

    const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(replaced).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
    expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32', '93.184.216.36/32'])
    expect(replaced!.metadata!.annotations).toMatchObject({
      'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
      'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.35/32',
      'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
    })
  })

  it('leaves a terminating policy unwritten on the DNS-failure prune and still rejects with the DNS error', async () => {
    const logs = captureRunLaneNetworkPolicyLogs()
    try {
      const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
      const activePreviousExpiresAt = new Date(Date.now() + 5 * 60 * 1000).toISOString()
      networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
        metadata: {
          resourceVersion: 'rv-1',
          deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
          annotations: {
            'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.36/32',
            'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32,93.184.216.35/32',
            'clerum.io/runtime-http-egress-previous-expires-at': activePreviousExpiresAt,
            'clerum.io/runtime-http-egress-previous-cidr-expiries': JSON.stringify({
              '93.184.216.34/32': '2000-01-01T00:00:00.000Z',
              '93.184.216.35/32': activePreviousExpiresAt,
            }),
          },
        },
      })
      const reconciler = new WorkflowReconciler(
        makeDeps({
          networkingApi: networkingApi as never,
          resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
          config: { ...makeConfig(), enableSnippetRuntime: true } as never,
        })
      )
      const spec = makeSpec({
        agent: undefined,
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
              capabilities: {
                http: { allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })

      await expect(
        reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
      ).rejects.toThrow('ENOTFOUND')

      expect(
        networkingApi.readNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]) => (arg as { name: string }).name === 'test-wf-snippet-runner-egress'
        ).length
      ).toBeGreaterThanOrEqual(2)
      expect(
        logs.entries.some(
          entry =>
            entry.level === 'warn' &&
            entry.reason === 'terminating' &&
            String(entry.msg).includes('NetworkPolicy "test-wf-snippet-runner-egress"')
        )
      ).toBe(true)
      // The refresh rethrows the DNS error, so the prune's own error line is
      // the only record that a policy still carries the CIDRs it meant to drop.
      expect(
        logs.entries.some(
          entry =>
            entry.level === 'error' &&
            String(entry.msg).includes('DNS-failure prune left a NetworkPolicy pending a retry')
        )
      ).toBe(true)
      expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
      expect(networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
    } finally {
      logs.restore()
    }
  })

  it('keeps the CIDRs of a policy owned by another controller out of the DNS-failure prune', async () => {
    const logs = captureRunLaneNetworkPolicyLogs()
    try {
      const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
      const livePolicies: Record<string, k8s.V1NetworkPolicy> = {
        // Ours: the rollover to .35 is done and its .34 overlap has expired.
        'test-wf-coord-to-wrc': {
          metadata: {
            name: 'test-wf-coord-to-wrc',
            resourceVersion: 'rv-coord',
            annotations: {
              'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
              'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
              'clerum.io/runtime-http-egress-previous-expires-at': '2000-01-01T00:00:00.000Z',
            },
          },
        },
        // Another controller's: frozen on the CIDR DNS no longer returns.
        'test-wf-snippet-runner-egress': {
          metadata: {
            name: 'test-wf-snippet-runner-egress',
            resourceVersion: 'rv-runner',
            ownerReferences: [
              {
                apiVersion: 'apps/v1',
                kind: 'Deployment',
                name: 'another-controller',
                uid: 'foreign-owner-uid',
                controller: true,
              },
            ],
            annotations: {
              'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
            },
          },
        },
      }
      networkingApi.readNamespacedNetworkPolicy.mockImplementation(
        async ({ name }: { name: string }) => {
          const found = livePolicies[name]
          if (!found) throw { code: 404 }
          return structuredClone(found)
        }
      )
      const reconciler = new WorkflowReconciler(
        makeDeps({
          networkingApi: networkingApi as never,
          resolveRuntimeHttpEgressCidrs: vi.fn().mockRejectedValue(new Error('ENOTFOUND')),
          config: { ...makeConfig(), enableSnippetRuntime: true } as never,
        })
      )
      const spec = makeSpec({
        agent: undefined,
        // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
        coordinatorImage: 'registry.example/custom-coordinator:1',
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
              capabilities: {
                http: { allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })

      await expect(
        reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
      ).rejects.toThrow('ENOTFOUND')

      // Liveness: the prune read both policies and applied the runner, which
      // the ownership branch refused.
      const readNames = networkingApi.readNamespacedNetworkPolicy.mock.calls.map(
        ([arg]) => (arg as { name: string }).name
      )
      expect(readNames).toEqual(
        expect.arrayContaining(['test-wf-coord-to-wrc', 'test-wf-snippet-runner-egress'])
      )
      expect(
        logs.entries.some(
          entry =>
            entry.level === 'warn' &&
            entry.reason === 'owner-reference-mismatch' &&
            String(entry.msg).includes('test-wf-snippet-runner-egress')
        )
      ).toBe(true)

      const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
      expect(replaced.map(arg => arg.name)).toEqual(['test-wf-coord-to-wrc'])
      expect(publicHttpEgressCidrs(replaced[0]!.body)).toEqual(['93.184.216.35/32'])
      expect(replaced[0]!.body.metadata?.annotations).toMatchObject({
        'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
      })
      expect(
        replaced[0]!.body.metadata?.annotations?.['clerum.io/runtime-http-egress-previous-cidrs']
      ).toBeUndefined()
    } finally {
      logs.restore()
    }
  })

  it('creates runtime HTTP egress policies without previous overlap on first refresh', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    const created = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(created).toBeDefined()
    expect(publicHttpEgressCidrs(created!)).toEqual(['93.184.216.34/32'])
    expect(created!.metadata!.annotations).toMatchObject({
      'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.34/32',
    })
    expect(
      created!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']
    ).toBeUndefined()
    expect(
      created!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
    ).toBeUndefined()
  })

  it('drops expired runtime HTTP egress overlap CIDRs on refresh', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: {
        resourceVersion: 'rv-1',
        annotations: {
          'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
          'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
          'clerum.io/runtime-http-egress-previous-expires-at': '2000-01-01T00:00:00.000Z',
        },
      },
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.35/32']),
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(replaced).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
    expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.35/32'])
    expect(
      replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']
    ).toBeUndefined()
    expect(
      replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-expires-at']
    ).toBeUndefined()
  })

  it('drops invalid, private, or untrusted-future runtime HTTP egress overlap CIDRs from annotations', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: {
        resourceVersion: 'rv-1',
        annotations: {
          'clerum.io/runtime-http-egress-current-cidrs':
            '10.0.0.5/32,93.184.216.34/32,0.0.0.0/0,garbage',
          'clerum.io/runtime-http-egress-previous-cidrs':
            '192.168.1.5/32,93.184.216.0/24,93.184.216.33/32',
          'clerum.io/runtime-http-egress-previous-expires-at': '2999-01-01T00:00:00.000Z',
        },
      },
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.35/32']),
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const spec = makeSpec({
      agent: undefined,
      runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
      steps: [
        {
          id: 'snippet',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
            capabilities: {
              http: { allowedHosts: ['api.example.com'] },
            },
          },
        },
      ],
    })

    await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
      'sandbox-recipes',
      'test-wf',
      'uid-123',
      spec
    )

    const replaced = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-snippet-runner-egress'
    )?.[0].body
    expect(replaced).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-snippet-runner-egress')
    expect(publicHttpEgressCidrs(replaced!)).toEqual(['93.184.216.34/32', '93.184.216.35/32'])
    expect(replaced!.metadata!.annotations!['clerum.io/runtime-http-egress-previous-cidrs']).toBe(
      '93.184.216.34/32'
    )
  })

  it('rejects snippet HTTP hosts that are not declared in runtimeEgress', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchText("https://api.example.com/data")',
              capabilities: {
                http: { allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('must be declared in spec.runtimeEgress.http.allowedHosts')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects public-web snippet HTTP capabilities that also declare exact allowedHosts', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        runtimeEgress: { http: { egressClass: 'public-web' } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchText("https://api.example.com/data")',
              capabilities: {
                http: { egressClass: 'public-web', allowedHosts: ['api.example.com'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('allowedHosts must be omitted when egressClass is public-web')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects snippet database workloads that are not declared in the recipe', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.postgres.query({ workload: "missing", database: "clerum" }, { sql: "select 1" })',
              capabilities: {
                postgres: { access: 'read', workloads: ['missing'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('references undeclared postgres workload "missing"')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects snippet database capabilities without explicit read or readWrite access', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
              capabilities: {
                postgres: { workloads: ['postgres'] },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('postgres capability must declare access read or readWrite')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects snippet MCP servers and tools outside the declared capability context', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        mcpServers: [{ id: 'mock-tools', endpoint: 'http://mock-tools.mcp-server.svc:3000/mcp' }],
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 1, b: 2 })',
              capabilities: {
                mcp: {
                  servers: ['mock-tools'],
                  allowedTools: { include: ['other-server__add'] },
                },
              },
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('must be scoped to an allowed server')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it.each([
    ['169.254.169.254', 'must not be an IP literal'],
    ['metadata.google.internal', 'must be a public DNS hostname'],
    ['metadata.goog', 'must be a public DNS hostname'],
    ['postgres.sandbox-recipes.svc.cluster.local', 'must be a public DNS hostname'],
    ['pod-1.sandbox-recipes.pod.cluster.local', 'must be a public DNS hostname'],
  ])(
    'rejects non-public runtimeEgress HTTP host %s before creating workflow pods',
    async (host, message) => {
      const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
      const reconciler = new WorkflowReconciler(
        makeDeps({
          coreApi: coreApi as never,
          config: {
            ...makeConfig(),
            enableCustomCoordinatorImage: true,
            allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
          } as never,
        })
      )

      const result = await reconciler.reconcile(
        'test-wf',
        'uid-123',
        'sandbox-recipes',
        makeSpec({
          agent: undefined,
          coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
          runtimeEgress: { http: { allowedHosts: [host] } },
          steps: [{ id: 'prepare' }],
        })
      )

      expect(result.workflowPhase).toBe('failed')
      expect(result.message).toContain(message)
      expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
    }
  )

  it('rejects too many runtimeEgress HTTP hosts before DNS resolution', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        runtimeEgress: {
          http: {
            allowedHosts: Array.from({ length: 21 }, (_, index) => `api-${index}.example.com`),
          },
        },
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('runtimeEgress.http.allowedHosts must contain at most 20')
    expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('replaces existing NetworkPolicies so removed HTTP egress cannot persist', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
      metadata: { name: 'test-wf-coord-to-wrc', resourceVersion: 'rv-1' },
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalled()
    const coordPolicy = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.name === 'test-wf-coord-to-wrc'
    )?.[0].body
    expect(coordPolicy).toBeDefined()
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
    expect(coordPolicy!.metadata!.resourceVersion).toBe('rv-1')
    expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(false)
  })

  it('prunes legacy recipe MCP server broad internet egress policies during reconcile', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({ mcpServers: ['redis-mcp'] })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'test-wf-mcp-servers-egress-internet',
      namespace: 'mcp-server',
    })
    expect(
      networkingApi.createNamespacedNetworkPolicy.mock.calls.some(
        ([arg]) => arg.body?.metadata?.name === 'test-wf-mcp-servers-egress-internet'
      )
    ).toBe(false)

    networkingApi.deleteNamespacedNetworkPolicy.mockClear()
    networkingApi.listNamespacedNetworkPolicy.mockClear()
    const second = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({ mcpServers: ['redis-mcp'] })
    )
    expect(second.workflowPhase).not.toBe('failed')
    expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalled()
    expect(networkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalledWith({
      name: 'test-wf-mcp-servers-egress-internet',
      namespace: 'mcp-server',
    })
  })

  // R4-L4: a 403 does not mark the policy gone; the next DELETE waits for the
  // end of its backoff window (60 s after the first failure).
  it('G3: does not skip the legacy internet NP after DELETE 403, and retries it once its backoff window ends', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
      if (name === 'test-wf-mcp-servers-egress-internet' && namespace === 'mcp-server') {
        throw { code: 403 }
      }
      return {}
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )
    const legacyDeletes = () =>
      networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) =>
          arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
      )
    const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
    vi.useFakeTimers({ toFake: ['Date'] })
    try {
      vi.setSystemTime(t0)
      const first = await reconciler.reconcile(
        'test-wf',
        'uid-123',
        'sandbox-recipes',
        makeSpec({ mcpServers: ['redis-mcp'] })
      )
      expect(legacyDeletes()).toHaveLength(1)
      expect(
        (first.networkPolicyOwnershipConditions ?? []).filter(
          c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
        )
      ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])

      vi.setSystemTime(t0 + 30_000)
      const inside = await reconciler.reconcile(
        'test-wf',
        'uid-123',
        'sandbox-recipes',
        makeSpec({ mcpServers: ['redis-mcp'] })
      )
      // Liveness witness: the pass reached the prune (the legacy fact is
      // published), and inside the window it sent no DELETE.
      expect(
        (inside.networkPolicyOwnershipConditions ?? []).filter(
          c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
        )
      ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])
      expect(legacyDeletes()).toHaveLength(1)
      expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
        30_000
      )

      vi.setSystemTime(t0 + 60_000)
      await reconciler.reconcile(
        'test-wf',
        'uid-123',
        'sandbox-recipes',
        makeSpec({ mcpServers: ['redis-mcp'] })
      )
      expect(legacyDeletes()).toHaveLength(2)
      // The second failure doubles the window.
      expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(
        120_000
      )
    } finally {
      vi.useRealTimers()
    }
  })

  it('R4-L4: doubles the legacy DELETE backoff up to 1 h, and a 404 ends it', async () => {
    const LEGACY = 'test-wf-mcp-servers-egress-internet'
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    let legacyAnswer = { code: 403 }
    networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name, namespace }) => {
      if (name === LEGACY && namespace === 'mcp-server') throw legacyAnswer
      return {}
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )
    const legacyDeletes = () =>
      networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === LEGACY && arg.namespace === 'mcp-server'
      )
    vi.useFakeTimers({ toFake: ['Date'] })
    try {
      let now = new Date('2026-09-23T10:00:00.000Z').getTime()
      const windows: number[] = []
      for (let attempt = 0; attempt < 8; attempt++) {
        vi.setSystemTime(now)
        expect(
          await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')
        ).toBe('pending')
        const delay = reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')
        windows.push(delay)
        now += delay
      }

      expect(windows).toEqual([
        60_000, 120_000, 240_000, 480_000, 960_000, 1_920_000, 3_600_000, 3_600_000,
      ])
      expect(legacyDeletes()).toHaveLength(8)

      legacyAnswer = { code: 404 }
      vi.setSystemTime(now)
      expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
        'removed'
      )
      expect(reconciler.legacyMcpServersInternetEgressRetryDelayMs('test-wf', 'uid-123')).toBe(0)
      expect(legacyDeletes()).toHaveLength(9)
      // Observed gone: a later retry answers from the gone set, with no DELETE.
      expect(await reconciler.retryLegacyMcpServersInternetEgressDelete('test-wf', 'uid-123')).toBe(
        'removed'
      )
      expect(legacyDeletes()).toHaveLength(9)
    } finally {
      vi.useRealTimers()
    }
  })

  it('R4-L4: the finalizer deletes the unlabelled legacy internet policy by name in the mcp-server namespace', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    // The label sweep finds nothing: the legacy policy predates the labels.
    networkingApi.listNamespacedNetworkPolicy.mockResolvedValue({ items: [] })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )

    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes', makeSpec(), 'uid-123')

    // Liveness witness: the fixed run-lane list was deleted by name.
    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'test-wf-mcp-host-to-servers',
      namespace: 'sandbox-recipes',
    })
    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'test-wf-mcp-servers-egress-internet',
      namespace: 'mcp-server',
    })
  })

  it('clears the legacy internet NP process set on recipe delete so a recreate can delete again', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({ mcpServers: ['redis-mcp'] })
    )
    networkingApi.deleteNamespacedNetworkPolicy.mockClear()
    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes', makeSpec(), 'uid-123')
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({ mcpServers: ['redis-mcp'] })
    )
    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'test-wf-mcp-servers-egress-internet',
      namespace: 'mcp-server',
    })
  })

  it('keys the legacy internet NP ledger by recipe uid, so a recreated recipe deletes again', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
      })
    )
    const legacyDeletes = () =>
      networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) =>
          arg.name === 'test-wf-mcp-servers-egress-internet' && arg.namespace === 'mcp-server'
      )
    const reconcileWithUid = (uid: string) =>
      reconciler.reconcile(
        'test-wf',
        uid,
        'sandbox-recipes',
        makeSpec({ mcpServers: ['redis-mcp'] })
      )

    await reconcileWithUid('uid-a')
    await reconcileWithUid('uid-a')
    // Witness: the ledger suppresses the repeat for the same uid.
    expect(legacyDeletes()).toHaveLength(1)

    // Same name, new uid, and no reconcileDelete seen by this process (another
    // replica handled the delete): a new identity must not be suppressed.
    await reconcileWithUid('uid-b')
    expect(legacyDeletes()).toHaveLength(2)
  })

  describe('B2 desired-set NetworkPolicy prune', () => {
    const RECIPE = 'test-wf'
    const SANDBOX = 'sandbox-recipes'
    const TWO_TERM_SELECTOR = `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`
    const CODEX_PROXY = `${RECIPE}-mcp-host-to-codex-proxy`
    const GROK_PROXY = `${RECIPE}-mcp-host-to-grok-proxy`
    const COORD_TO_WRC = `${RECIPE}-coord-to-wrc`
    const COORD_TO_MCP_HOST = `${RECIPE}-coord-to-mcp-host`
    const WRC_LABELS = {
      'clerum.io/recipe': RECIPE,
      'clerum.io/managed-by': 'wrc',
    }

    function ineligibleProjection(
      overrides: Partial<CodexExecutionProjection> = {}
    ): CodexExecutionProjection {
      return {
        targets: [],
        eligibleTargets: [],
        derivedScopes: [],
        requiresCodexProxyEgress: false,
        driftHashInput: '',
        catalogContentHash: null,
        catalogRevision: null,
        connectionRevision: null,
        eligibility: 'ineligible',
        reason: 'static_only',
        ...overrides,
      }
    }

    function seedLivePolicy(
      apiserver: ReturnType<typeof makeApiserverNetworkingApi>,
      name: string,
      labels: Record<string, string>,
      namespace = SANDBOX
    ): void {
      apiserver.live.set(apiserver.key(namespace, name), {
        apiVersion: 'networking.k8s.io/v1',
        kind: 'NetworkPolicy',
        metadata: { name, namespace, labels },
        spec: { podSelector: {}, policyTypes: ['Egress'] },
      })
    }

    function listCalls(
      api: ReturnType<typeof makeApiserverNetworkingApi>['api']
    ): Array<{ namespace: string; labelSelector?: string }> {
      return api.listNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
    }

    function sandboxDeletes(
      api: ReturnType<typeof makeApiserverNetworkingApi>['api']
    ): Array<{ name: string; namespace: string }> {
      return api.deleteNamespacedNetworkPolicy.mock.calls
        .map(([arg]) => arg)
        .filter(arg => arg.namespace === SANDBOX)
    }

    async function applyPolicies(
      reconciler: WorkflowReconciler,
      opts: {
        codex?: Partial<CodexExecutionProjection>
        grok?: Partial<CodexExecutionProjection> & { requiresGrokProxyEgress?: boolean }
        awaitsTriggeredRun?: boolean
        eagerSdkMcpHost?: boolean
      } = {}
    ): Promise<{ retryPending: boolean }> {
      const spec = makeSpec()
      const runtime = deriveWorkflowRuntimePlan(spec, {
        recipeName: RECIPE,
        runtimeScopeRecipeName: RECIPE,
        workflowRunId: opts.awaitsTriggeredRun ? undefined : 'run-1',
      })
      const codex = ineligibleProjection({
        requiresCodexProxyEgress: false,
        ...opts.codex,
      })
      const grok = {
        ...ineligibleProjection(),
        requiresCodexProxyEgress: false,
        requiresGrokProxyEgress: false,
        ...opts.grok,
      }
      return (await (
        reconciler as unknown as {
          applyWorkflowNetworkPolicies: (
            recipeName: string,
            recipeUid: string,
            spec: WorkflowRecipeSpec,
            runtime: ReturnType<typeof deriveWorkflowRuntimePlan>,
            awaitsTriggeredRun: boolean,
            codexProjection: CodexExecutionProjection,
            eagerSdkMcpHost: boolean,
            grokProjection: typeof grok
          ) => Promise<{ retryPending: boolean }>
        }
      ).applyWorkflowNetworkPolicies(
        RECIPE,
        'uid-123',
        spec,
        runtime,
        opts.awaitsTriggeredRun === true,
        codex,
        opts.eagerSdkMcpHost === true,
        grok
      )) as { retryPending: boolean }
    }

    function expectListWitness(
      api: ReturnType<typeof makeApiserverNetworkingApi>['api'],
      times = 1
    ): void {
      expect(listCalls(api)).toEqual(
        Array.from({ length: times }, () => ({
          namespace: SANDBOX,
          labelSelector: TWO_TERM_SELECTOR,
        }))
      )
    }

    /**
     * Liveness witness for the negative prune tests: a catalog member that is not
     * desired sits in the same LIST and must be deleted. If the prune loop never
     * ran, "X was not deleted" would hold anyway; the sibling's delete proves the
     * loop iterated over the listed items.
     */
    function seedPrunableSibling(apiserver: ReturnType<typeof makeApiserverNetworkingApi>): void {
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
    }

    function expectPrunableSiblingDeleted(
      apiserver: ReturnType<typeof makeApiserverNetworkingApi>
    ): void {
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
      expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
    }

    it('does not delete a listed name that is outside the run-lane catalog', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
      seedPrunableSibling(apiserver)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler)

      expectListWitness(apiserver.api)
      expectPrunableSiblingDeleted(apiserver)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(`${RECIPE}-obsolete-wrc`)
      expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
    })

    it('G1: reserved coordinator-to-gfs survives a run-lane apply', async () => {
      const apiserver = makeApiserverNetworkingApi()
      const gfsName = `${RECIPE}-coordinator-to-gfs`
      seedLivePolicy(apiserver, gfsName, WRC_LABELS)
      seedLivePolicy(apiserver, `${RECIPE}-obsolete-wrc`, WRC_LABELS)
      seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
      seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
        'clerum.io/recipe': RECIPE,
        'clerum.io/managed-by': 'workflow-recipes',
      })
      seedLivePolicy(apiserver, `${RECIPE}-outer-unknown-wrc`, WRC_LABELS)
      seedLivePolicy(apiserver, 'other-recipe-coordinator-to-gfs', {
        'clerum.io/recipe': 'other-recipe',
        'clerum.io/managed-by': 'wrc',
      })
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler)

      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
      expect(apiserver.live.has(apiserver.key(SANDBOX, gfsName))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-obsolete-wrc`))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-outer-unknown-wrc`))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, 'other-recipe-coordinator-to-gfs'))).toBe(
        true
      )
      expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)
    })

    it('deletes a seeded leftover Grok proxy and does not name-delete when unseeded', async () => {
      const seeded = makeApiserverNetworkingApi()
      seedLivePolicy(seeded, GROK_PROXY, WRC_LABELS)
      const seededReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: seeded.api as never })
      )
      await applyPolicies(seededReconciler)
      expectListWitness(seeded.api)
      expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([GROK_PROXY])

      const unseeded = makeApiserverNetworkingApi()
      const unseededReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: unseeded.api as never })
      )
      await applyPolicies(unseededReconciler)
      expectListWitness(unseeded.api)
      expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(GROK_PROXY)
    })

    it('deletes a seeded leftover Codex proxy and does not name-delete when unseeded', async () => {
      const seeded = makeApiserverNetworkingApi()
      seedLivePolicy(seeded, CODEX_PROXY, WRC_LABELS)
      const seededReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: seeded.api as never })
      )
      await applyPolicies(seededReconciler)
      expect(sandboxDeletes(seeded.api).map(d => d.name)).toEqual([CODEX_PROXY])

      const unseeded = makeApiserverNetworkingApi()
      const unseededReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: unseeded.api as never })
      )
      await applyPolicies(unseededReconciler)
      expect(sandboxDeletes(unseeded.api).map(d => d.name)).not.toContain(CODEX_PROXY)
    })

    it('c2: awaiting-trigger prunes a run-only leftover that apply(false) would keep', async () => {
      const leftover = `${RECIPE}-wrc-to-artifact-reader`
      const awaiting = makeApiserverNetworkingApi()
      seedLivePolicy(awaiting, leftover, WRC_LABELS)
      const awaitingReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: awaiting.api as never })
      )
      await applyPolicies(awaitingReconciler, { awaitsTriggeredRun: true })
      expect(sandboxDeletes(awaiting.api).map(d => d.name)).toEqual([leftover])

      const running = makeApiserverNetworkingApi()
      seedLivePolicy(running, leftover, WRC_LABELS)
      const runningReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: running.api as never })
      )
      await applyPolicies(runningReconciler, { awaitsTriggeredRun: false })
      expect(sandboxDeletes(running.api).map(d => d.name)).not.toContain(leftover)
      expect(running.live.has(running.key(SANDBOX, leftover))).toBe(true)
    })

    it('does not delete in the sandbox namespace when the LIST is empty', async () => {
      const apiserver = makeApiserverNetworkingApi()
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler)

      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api)).toEqual([])
    })

    it('does not delete a listed policy that is in the desired set', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, COORD_TO_WRC, WRC_LABELS)
      seedPrunableSibling(apiserver)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler)

      expectPrunableSiblingDeleted(apiserver)
      expect(sandboxDeletes(apiserver.api).filter(d => d.name === COORD_TO_WRC)).toEqual([])
      expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_WRC))).toBe(true)
    })

    it('does not prune an uncatalogued policy carrying the recipe label pair; LIST is issued once', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, `${RECIPE}-oauth-broker-egress`, {
        'clerum.io/recipe': RECIPE,
        'clerum.io/managed-by': 'workflow-recipes',
      })
      seedPrunableSibling(apiserver)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler)

      expectListWitness(apiserver.api)
      expectPrunableSiblingDeleted(apiserver)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(
        `${RECIPE}-oauth-broker-egress`
      )
      expect(apiserver.live.has(apiserver.key(SANDBOX, `${RECIPE}-oauth-broker-egress`))).toBe(true)
    })

    it('keeps an unlisted-as-desired proxy while eligibility is uncertain, and deletes the twin', async () => {
      const uncertain = makeApiserverNetworkingApi()
      seedLivePolicy(uncertain, CODEX_PROXY, WRC_LABELS)
      const debugLog = captureLogger('debug')
      try {
        const uncertainReconciler = new WorkflowReconciler(
          makeDeps({ networkingApi: uncertain.api as never })
        )
        await applyPolicies(uncertainReconciler, {
          codex: { eligibility: 'uncertain', requiresCodexProxyEgress: false, reason: 'forbidden' },
        })
        // The skip names why the proxy survives.
        expect(debugLog).toHaveBeenCalledWith(
          'Skipping run-lane NP prune',
          expect.objectContaining({ policy: CODEX_PROXY, reason: 'uncertain' })
        )
      } finally {
        debugLog.mockRestore()
      }
      expectListWitness(uncertain.api)
      expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(CODEX_PROXY)
      expect(uncertain.live.has(uncertain.key(SANDBOX, CODEX_PROXY))).toBe(true)

      const decided = makeApiserverNetworkingApi()
      seedLivePolicy(decided, CODEX_PROXY, WRC_LABELS)
      const decidedReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: decided.api as never })
      )
      await applyPolicies(decidedReconciler, {
        codex: { eligibility: 'ineligible', requiresCodexProxyEgress: false },
      })
      expectListWitness(decided.api)
      expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(CODEX_PROXY)
    })

    it('keeps a desired live Codex proxy and deletes an undesired Grok proxy only on the first pass', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, CODEX_PROXY, WRC_LABELS)
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler, {
        codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
      })
      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toEqual([GROK_PROXY])
      expect(apiserver.live.has(apiserver.key(SANDBOX, CODEX_PROXY))).toBe(true)
      expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(false)

      apiserver.api.listNamespacedNetworkPolicy.mockClear()
      apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
      await applyPolicies(reconciler, {
        codex: { eligibility: 'eligible', requiresCodexProxyEgress: true, reason: 'granted' },
      })
      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api)).toEqual([])
    })

    it('keeps an unlisted-as-desired Grok proxy while eligibility is uncertain, and deletes the twin', async () => {
      const uncertain = makeApiserverNetworkingApi()
      seedLivePolicy(uncertain, GROK_PROXY, WRC_LABELS)
      const uncertainReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: uncertain.api as never })
      )
      await applyPolicies(uncertainReconciler, {
        grok: { eligibility: 'uncertain', requiresGrokProxyEgress: false, reason: 'forbidden' },
      })
      expectListWitness(uncertain.api)
      expect(sandboxDeletes(uncertain.api).map(d => d.name)).not.toContain(GROK_PROXY)
      expect(uncertain.live.has(uncertain.key(SANDBOX, GROK_PROXY))).toBe(true)

      const decided = makeApiserverNetworkingApi()
      seedLivePolicy(decided, GROK_PROXY, WRC_LABELS)
      const decidedReconciler = new WorkflowReconciler(
        makeDeps({ networkingApi: decided.api as never })
      )
      await applyPolicies(decidedReconciler, {
        grok: { eligibility: 'ineligible', requiresGrokProxyEgress: false },
      })
      expectListWitness(decided.api)
      expect(sandboxDeletes(decided.api).map(d => d.name)).toContain(GROK_PROXY)
    })

    it('does not prune a catalog-named recipes-lane policy (two-term selector)', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, COORD_TO_MCP_HOST, {
        'clerum.io/recipe': RECIPE,
        'clerum.io/managed-by': 'workflow-recipes',
      })
      seedPrunableSibling(apiserver)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler, { awaitsTriggeredRun: true })

      expectListWitness(apiserver.api)
      expectPrunableSiblingDeleted(apiserver)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
      expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
    })

    it('deletes a leftover coord-to-mcp-host when awaiting trigger', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler, { awaitsTriggeredRun: true })

      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(COORD_TO_MCP_HOST)
      expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(false)
    })

    it('R1-M1: does not prune a catalog policy that carries an ownerReference (apply refuses it too)', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, COORD_TO_MCP_HOST, WRC_LABELS)
      const seeded = apiserver.live.get(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))!
      seeded.metadata = {
        ...seeded.metadata,
        ownerReferences: [
          {
            apiVersion: 'apps/v1',
            kind: 'Deployment',
            name: 'someone-else',
            uid: 'uid-deployment',
            controller: true,
          },
        ],
      }
      seedPrunableSibling(apiserver)
      const warnLog = captureLogger('warn')
      try {
        const reconciler = new WorkflowReconciler(
          makeDeps({ networkingApi: apiserver.api as never })
        )

        await applyPolicies(reconciler, { awaitsTriggeredRun: true })

        expectListWitness(apiserver.api)
        // Liveness witness: the WRC-owned sibling in the same LIST was pruned.
        expectPrunableSiblingDeleted(apiserver)
        expect(sandboxDeletes(apiserver.api).map(d => d.name)).not.toContain(COORD_TO_MCP_HOST)
        expect(apiserver.live.has(apiserver.key(SANDBOX, COORD_TO_MCP_HOST))).toBe(true)
        expect(warnLog).toHaveBeenCalledWith('Skipping run-lane NP prune', {
          recipe: RECIPE,
          policy: COORD_TO_MCP_HOST,
          reason: 'owner-reference-mismatch',
        })
      } finally {
        warnLog.mockRestore()
      }
    })

    it('eager SDK path still prunes a leftover Grok proxy', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      await applyPolicies(reconciler, { awaitsTriggeredRun: true, eagerSdkMcpHost: true })

      expectListWitness(apiserver.api)
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
    })

    // A prune DELETE that did not land is prune-pending, not a pending retry:
    // it publishes PrunePending and does not requeue (R3-L3).
    it('marks the prune pending, not a retry, when a leftover prune DELETE fails', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
        async ({ name, namespace }: { name: string; namespace: string }) => {
          if (name === GROK_PROXY) throw { code: 403, message: 'forbidden' }
          if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
          return {}
        }
      )
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      const summary = await applyPolicies(reconciler)
      expect(summary.prune).toBe('pending')
      expect(summary.retryPending).toBe(false)
      expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
    })

    it.each([
      { code: 404, prune: 'converged' },
      { code: 409, prune: 'pending' },
      { code: 500, prune: 'pending' },
    ])('prune DELETE answering $code leaves the prune $prune', async ({ code, prune }) => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
        async ({ name, namespace }: { name: string; namespace: string }) => {
          if (name === GROK_PROXY) throw { code }
          if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
          return {}
        }
      )
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      const summary = await applyPolicies(reconciler)

      // Liveness witness: the prune reached the DELETE for the leftover.
      expect(sandboxDeletes(apiserver.api).map(d => d.name)).toContain(GROK_PROXY)
      expect(summary.prune).toBe(prune)
      expect(summary.retryPending).toBe(false)
    })

    it('logs prune failures as an error object, never as a pre-flattened message string', async () => {
      const errorLog = captureLogger('error')
      try {
        // A client exception whose message embeds the response body and headers: the
        // logger only redacts those when it receives the error itself.
        const failure = Object.assign(new Error('HTTP-Code: 403 Body: "{}" Headers: {}'), {
          code: 403,
        })
        const apiserver = makeApiserverNetworkingApi()
        seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
        apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
          async ({ name, namespace }: { name: string; namespace: string }) => {
            if (name === GROK_PROXY) throw failure
            if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
            return {}
          }
        )
        apiserver.api.listNamespacedNetworkPolicy.mockRejectedValueOnce(failure)
        const reconciler = new WorkflowReconciler(
          makeDeps({ networkingApi: apiserver.api as never })
        )

        await applyPolicies(reconciler) // LIST fails
        await applyPolicies(reconciler) // LIST ok, DELETE of the leftover fails

        const fields = (message: string) =>
          errorLog.mock.calls.find(([msg]) => msg === message)?.[1] as
            | Record<string, unknown>
            | undefined
        for (const message of [
          'Run-lane NP prune LIST failed; apply outcome is kept',
          'Run-lane NP prune delete failed',
        ]) {
          expect(fields(message), message).toBeDefined()
          expect(fields(message)?.err).toBe(failure)
          expect(fields(message)).not.toHaveProperty('error')
        }
      } finally {
        errorLog.mockRestore()
      }
    })

    it('marks the legacy policy pending, apart from the prune, when the legacy internet NP DELETE fails', async () => {
      const apiserver = makeApiserverNetworkingApi()
      apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
        async ({ name, namespace }: { name: string; namespace: string }) => {
          if (name === `${RECIPE}-mcp-servers-egress-internet`) {
            throw { code: 403, message: 'forbidden' }
          }
          if (!apiserver.live.delete(apiserver.key(namespace, name))) throw { code: 404 }
          return {}
        }
      )
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      const summary = await applyPolicies(reconciler)
      // Liveness witness: the legacy delete was attempted.
      expect(
        apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
        )
      ).toHaveLength(1)
      // R4-L3: the legacy delete is its own fact; the catalog prune converged.
      expect(summary.legacy).toBe('pending')
      expect(summary.prune).toBe('converged')
      expect(summary.retryPending).toBe(false)
    })

    // R4-L6: a failed LIST evaluated nothing. It is not a DELETE that did
    // not land, so it is its own prune fact, `unevaluated`.
    it('R4-L6: marks the prune unevaluated, not pending, and keeps the apply outcome when the prune LIST fails', async () => {
      const apiserver = makeApiserverNetworkingApi()
      seedLivePolicy(apiserver, GROK_PROXY, WRC_LABELS)
      apiserver.api.listNamespacedNetworkPolicy.mockRejectedValue({ code: 500, message: 'boom' })
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: apiserver.api as never }))

      const summary = await applyPolicies(reconciler)
      expect(summary.prune).toBe('unevaluated')
      expect(summary.retryPending).toBe(false)
      expect(sandboxDeletes(apiserver.api)).toEqual([])
      expect(apiserver.live.has(apiserver.key(SANDBOX, GROK_PROXY))).toBe(true)
      // The apply outcome survives: the legacy internet policy prune that runs
      // after the catalog prune is still attempted despite the failed LIST.
      expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalled()
      expect(
        apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]: [{ name: string }]) => arg.name === `${RECIPE}-mcp-servers-egress-internet`
        )
      ).toHaveLength(1)
    })
  })

  describe('B3(d) skip coordinator DELETE after same-pass 404', () => {
    const COORDINATOR = 'test-wf-coordinator'
    const MCP_HOST = 'test-wf-mcp-host'
    const SANDBOX = 'sandbox-recipes'

    function podReads(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
      return coreApi.readNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
    }

    function podDeletes(coreApi: ReturnType<typeof makeCoreApi>, name: string): number {
      return coreApi.deleteNamespacedPod.mock.calls.filter(([arg]) => arg.name === name).length
    }

    it('does not delete the coordinator when same-pass GET is 404', async () => {
      const coreApi = makeCoreApi(false)
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())

      expect(result.workflowPhase).not.toBe('failed')
      expect(podReads(coreApi, COORDINATOR)).toBe(2)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
    })

    it('G4: deletes the coordinator when GET 200 has an empty status.phase', async () => {
      const coreApi = makeCoreApi(false)
      coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
        if (name === COORDINATOR) return { metadata: { name }, status: {} }
        throw { code: 404 }
      })
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())

      expect(podReads(coreApi, COORDINATOR)).toBe(2)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
    })

    it('G4: a 403 on coordinator GET does not skip as 404', async () => {
      const coreApi = makeCoreApi(false)
      coreApi.readNamespacedPod.mockImplementation(async ({ name }: { name: string }) => {
        if (name === COORDINATOR) throw { code: 403, message: 'forbidden' }
        throw { code: 404 }
      })
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())

      expect(result.workflowPhase).toBe('failed')
      // The 403 stops the reconcile at the first coordinator read; a 403
      // mapped to absent would continue to a second read.
      expect(podReads(coreApi, COORDINATOR)).toBe(1)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
    })

    it('c2: does not delete a Running coordinator when workflowRunId is set', async () => {
      const coreApi = makeCoreApi(true)
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      await reconciler.reconcile(
        'test-wf',
        'uid-123',
        SANDBOX,
        makeSpec(),
        undefined,
        undefined,
        undefined,
        'run-1'
      )

      expect(podReads(coreApi, COORDINATOR)).toBe(3)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
    })

    it('deletes the coordinator when same-pass getPodPhase is Running', async () => {
      const coreApi = makeCoreApi(true)
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())

      expect(podReads(coreApi, COORDINATOR)).toBe(3)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(1)
      expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith({
        name: COORDINATOR,
        namespace: SANDBOX,
      })
    })

    it('neither reads nor deletes mcp-host when needsMcpHost is false', async () => {
      const coreApi = makeCoreApi(false)
      const reconciler = new WorkflowReconciler(
        makeDeps({
          coreApi: coreApi as never,
          config: {
            ...makeConfig(),
            enableSnippetRuntime: true,
            snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
          } as never,
        })
      )

      await reconciler.reconcile(
        'test-wf',
        'uid-123',
        SANDBOX,
        makeSpec({
          agent: undefined,
          steps: [{ id: 'snippet', run: snippetRun() }],
        })
      )

      expect(podReads(coreApi, MCP_HOST)).toBe(0)
      expect(podDeletes(coreApi, MCP_HOST)).toBe(0)
      // Snippet-only has needsMcpHost=false, so awaitsTriggeredRun is false and
      // B3(d) does not run. The coordinator is still read for crash recovery.
      expect(podReads(coreApi, COORDINATOR)).toBe(2)
      expect(podDeletes(coreApi, COORDINATOR)).toBe(0)
    })

    it('does not delete mcp-host when same-pass GET is 404', async () => {
      const coreApi = makeCoreApi(false)
      const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

      const result = await reconciler.reconcile('test-wf', 'uid-123', SANDBOX, makeSpec())

      expect(result.workflowPhase).not.toBe('failed')
      expect(podReads(coreApi, MCP_HOST)).toBe(2)
      expect(podDeletes(coreApi, MCP_HOST)).toBe(0)
    })
  })

  it('retries NetworkPolicy replace once when resourceVersion is stale', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const readCounts = new Map<string, number>()
    const coordResourceVersions: Array<string | undefined> = []
    networkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
      const count = readCounts.get(name) ?? 0
      readCounts.set(name, count + 1)
      return {
        metadata: {
          name,
          resourceVersion: name === 'test-wf-coord-to-wrc' && count === 1 ? 'rv-2' : 'rv-1',
        },
      }
    })
    networkingApi.replaceNamespacedNetworkPolicy.mockImplementation(async ({ name, body }) => {
      if (name === 'test-wf-coord-to-wrc') {
        coordResourceVersions.push(body.metadata?.resourceVersion)
        const coordCalls = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]) => arg.name === 'test-wf-coord-to-wrc'
        ).length
        if (coordCalls === 1) throw { code: 409 }
      }
      return {}
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        networkingApi: networkingApi as never,
        config: { ...makeConfig() } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'prepare', run: snippetRun() }],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    const coordCalls = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.filter(
      ([arg]) => arg.name === 'test-wf-coord-to-wrc'
    )
    expect(coordCalls).toHaveLength(2)
    expect(coordResourceVersions).toEqual(['rv-1', 'rv-2'])
    expect(readCounts.get('test-wf-coord-to-wrc')).toBe(2)
    expect(createdPolicyNames(networkingApi)).not.toContain('test-wf-coord-to-wrc')
  })

  describe('run-lane NetworkPolicy convergence against apiserver-shaped live objects', () => {
    const RUN_LANE_POLICY_NAMES = [
      'sandbox-recipes/test-wf-coord-to-snippet-runner',
      'sandbox-recipes/test-wf-coord-to-snippet-runner-ingress',
      'sandbox-recipes/test-wf-coord-to-wrc',
      'sandbox-recipes/test-wf-snippet-runner-egress',
      'sandbox-recipes/test-wf-wrc-to-artifact-reader',
    ]

    function readPolicyNames(api: ReturnType<typeof makeApiserverNetworkingApi>['api']): string[] {
      return api.readNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name).sort()
    }

    function runtimeEgressSpec(): WorkflowRecipeSpec {
      return makeSpec({
        agent: undefined,
        runtimeEgress: { http: { allowedHosts: ['api.example.com'] } },
        steps: [
          {
            id: 'snippet',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.http.fetchJson("https://api.example.com/data")',
              capabilities: { http: { allowedHosts: ['api.example.com'] } },
            },
          },
        ],
      })
    }

    it('writes nothing on a second pass once every policy is live', async () => {
      const logs = captureRunLaneNetworkPolicyLogs()
      try {
        const { api, live } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        expect(first.workflowPhase).not.toBe('failed')
        expect([...live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)

        api.createNamespacedNetworkPolicy.mockClear()
        api.readNamespacedNetworkPolicy.mockClear()
        api.replaceNamespacedNetworkPolicy.mockClear()
        logs.entries.length = 0

        const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(second.workflowPhase).not.toBe('failed')
        const expectedNames = RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
        expect(readPolicyNames(api)).toEqual(expectedNames)
        expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        for (const name of expectedNames) {
          expect(
            logs.entries.some(entry =>
              String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
            )
          ).toBe(true)
        }
      } finally {
        logs.restore()
      }
    })

    // The factory always sends policyTypes and port protocols, so the reconcile
    // tests above never depend on the apiserver default tolerance. This drives the
    // run-lane apply directly with a policy that omits both.
    it('leaves a policy unchanged when the apiserver filled the defaults the desired omits', async () => {
      const logs = captureRunLaneNetworkPolicyLogs()
      try {
        const { api, live, key } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const applyNetworkPolicy = (
          reconciler as unknown as {
            applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<unknown>
          }
        ).applyNetworkPolicy.bind(reconciler)
        const name = 'test-wf-defaults-omitted'
        const desired = (): k8s.V1NetworkPolicy => ({
          apiVersion: 'networking.k8s.io/v1',
          kind: 'NetworkPolicy',
          metadata: {
            name,
            namespace: 'sandbox-recipes',
            labels: { 'clerum.io/managed-by': 'wrc', 'clerum.io/recipe': 'test-wf' },
          },
          spec: {
            podSelector: { matchLabels: { app: 'test-wf-coordinator' } },
            egress: [{ ports: [{ port: 443 }] }],
          },
        })

        expect(await applyNetworkPolicy(desired())).toEqual({ policy: name, action: 'created' })
        const stored = live.get(key('sandbox-recipes', name))
        expect(stored?.spec?.policyTypes).toEqual(['Ingress', 'Egress'])
        expect(stored?.spec?.egress?.[0]?.ports?.[0]?.protocol).toBe('TCP')

        api.createNamespacedNetworkPolicy.mockClear()
        api.readNamespacedNetworkPolicy.mockClear()
        api.replaceNamespacedNetworkPolicy.mockClear()
        logs.entries.length = 0

        expect(await applyNetworkPolicy(desired())).toEqual({ policy: name, action: 'unchanged' })
        expect(readPolicyNames(api)).toEqual([name])
        expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(
          logs.entries.filter(entry =>
            String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
          )
        ).toHaveLength(1)
      } finally {
        logs.restore()
      }
    })

    it('replaces exactly the drifted policy with its live resourceVersion', async () => {
      const { api, live, key } = makeApiserverNetworkingApi()
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
      const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

      await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
      const driftedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
      const drifted = live.get(driftedKey)
      expect(drifted).toBeDefined()
      drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { app: 'edited-by-hand' } } }
      const liveResourceVersion = drifted!.metadata!.resourceVersion

      api.createNamespacedNetworkPolicy.mockClear()
      api.readNamespacedNetworkPolicy.mockClear()
      api.replaceNamespacedNetworkPolicy.mockClear()

      const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

      expect(second.workflowPhase).not.toBe('failed')
      expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
      expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
      const [replaceArg] = api.replaceNamespacedNetworkPolicy.mock.calls[0]!
      expect(replaceArg.name).toBe('test-wf-coord-to-wrc')
      expect(replaceArg.body.metadata?.resourceVersion).toBe(liveResourceVersion)
      expect(live.get(driftedKey)!.spec!.podSelector).not.toEqual({
        matchLabels: { app: 'edited-by-hand' },
      })
      expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
    })

    it('refresh leaves a policy owned by another controller untouched and warns', async () => {
      const logs = captureRunLaneNetworkPolicyLogs()
      try {
        const { api, live, key } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(
          makeDeps({
            networkingApi: api as never,
            resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
            config: { ...makeConfig(), enableSnippetRuntime: true } as never,
          })
        )
        const spec = runtimeEgressSpec()
        await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
        const foreign = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
        expect(foreign).toBeDefined()
        foreign!.metadata = {
          ...foreign!.metadata,
          ownerReferences: [
            {
              apiVersion: 'apps/v1',
              kind: 'Deployment',
              name: 'another-controller',
              uid: 'foreign-owner-uid',
              controller: true,
            },
          ],
        }

        api.readNamespacedNetworkPolicy.mockClear()
        api.replaceNamespacedNetworkPolicy.mockClear()
        logs.entries.length = 0

        await expect(
          reconciler.refreshRuntimeHttpEgressNetworkPolicies(
            'sandbox-recipes',
            'test-wf',
            'uid-123',
            spec
          )
        ).resolves.toEqual({
          conflicts: [
            { policy: 'test-wf-snippet-runner-egress', reason: 'owner-reference-mismatch' },
          ],
          retryPending: false,
        })

        expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
        expect(
          api.replaceNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.name === 'test-wf-snippet-runner-egress'
          )
        ).toHaveLength(0)
        // Only the ownership branch emits this reason; a vanished-after-conflict
        // warning also names the policy and must not satisfy this witness.
        expect(
          logs.entries.some(
            entry =>
              entry.level === 'warn' &&
              entry.reason === 'owner-reference-mismatch' &&
              String(entry.msg).includes('test-wf-snippet-runner-egress')
          )
        ).toBe(true)
      } finally {
        logs.restore()
      }
    })

    it('refresh returns a pending retry for a terminating policy and leaves it unwritten', async () => {
      const { api, live, key } = makeApiserverNetworkingApi()
      const reconciler = new WorkflowReconciler(
        makeDeps({
          networkingApi: api as never,
          resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
          config: { ...makeConfig(), enableSnippetRuntime: true } as never,
        })
      )
      const spec = runtimeEgressSpec()
      await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
        'sandbox-recipes',
        'test-wf',
        'uid-123',
        spec
      )
      const terminating = live.get(key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
      expect(terminating).toBeDefined()
      terminating!.metadata = {
        ...terminating!.metadata,
        deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
      }

      api.readNamespacedNetworkPolicy.mockClear()
      api.replaceNamespacedNetworkPolicy.mockClear()

      await expect(
        reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
      ).resolves.toEqual({ conflicts: [], retryPending: true })

      expect(readPolicyNames(api)).toContain('test-wf-snippet-runner-egress')
      expect(
        api.replaceNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]) => arg.name === 'test-wf-snippet-runner-egress'
        )
      ).toHaveLength(0)
    })

    describe('runtime HTTP egress resolved-at', () => {
      // runtimeEgressSpec has no coordinator, so the snippet runner is the
      // only runtime HTTP egress policy.
      const RUNTIME_EGRESS_POLICIES = ['test-wf-snippet-runner-egress']
      const RESOLVED_AT = 'clerum.io/runtime-http-egress-resolved-at'
      const FIRST_PASS_AT = new Date('2026-09-23T10:00:00.000Z')
      const SECOND_PASS_AT = new Date('2026-09-23T10:05:00.000Z')

      afterEach(() => {
        vi.useRealTimers()
      })

      async function refreshTwice(resolveRuntimeHttpEgressCidrs: ReturnType<typeof vi.fn>) {
        const apiserver = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(
          makeDeps({
            networkingApi: apiserver.api as never,
            resolveRuntimeHttpEgressCidrs,
            config: { ...makeConfig(), enableSnippetRuntime: true } as never,
          })
        )
        const spec = runtimeEgressSpec()
        vi.useFakeTimers({ toFake: ['Date'] })
        vi.setSystemTime(FIRST_PASS_AT)
        await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
        for (const name of RUNTIME_EGRESS_POLICIES) {
          expect(
            apiserver.live.get(apiserver.key('sandbox-recipes', name))?.metadata?.annotations?.[
              RESOLVED_AT
            ]
          ).toBe(FIRST_PASS_AT.toISOString())
        }

        apiserver.api.createNamespacedNetworkPolicy.mockClear()
        apiserver.api.readNamespacedNetworkPolicy.mockClear()
        apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
        vi.setSystemTime(SECOND_PASS_AT)
        const second = await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
        return { ...apiserver, second }
      }

      it('keeps the live resolved-at and writes nothing when the resolved set is unchanged', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const resolve = vi.fn().mockResolvedValue(['93.184.216.34/32'])
          const { api, live, key, second } = await refreshTwice(resolve)

          expect(resolve).toHaveBeenCalledTimes(2)
          expect(readPolicyNames(api)).toEqual(expect.arrayContaining(RUNTIME_EGRESS_POLICIES))
          expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          for (const name of RUNTIME_EGRESS_POLICIES) {
            expect(
              live.get(key('sandbox-recipes', name))?.metadata?.annotations?.[RESOLVED_AT]
            ).toBe(FIRST_PASS_AT.toISOString())
          }
          // No write is also what a deferred retry produces. The Unchanged line,
          // which the first pass cannot emit because it creates the policy, and
          // the empty summary prove the second pass decided the policy converged.
          expect(second).toEqual({ conflicts: [], retryPending: false })
          for (const name of RUNTIME_EGRESS_POLICIES) {
            expect(
              logs.entries.filter(entry =>
                String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
              )
            ).toHaveLength(1)
          }
        } finally {
          logs.restore()
        }
      })

      it('stamps the current time on a policy that is missing although its sibling is unchanged', async () => {
        const apiserver = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(
          makeDeps({
            networkingApi: apiserver.api as never,
            resolveRuntimeHttpEgressCidrs: vi.fn().mockResolvedValue(['93.184.216.34/32']),
            config: { ...makeConfig(), enableSnippetRuntime: true } as never,
          })
        )
        // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
        const spec: WorkflowRecipeSpec = {
          ...runtimeEgressSpec(),
          coordinatorImage: 'registry.example/custom-coordinator:1',
        }
        vi.useFakeTimers({ toFake: ['Date'] })
        vi.setSystemTime(FIRST_PASS_AT)
        await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )
        const coordKey = apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
        const runnerKey = apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress')
        for (const policyKey of [coordKey, runnerKey]) {
          expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(
            FIRST_PASS_AT.toISOString()
          )
        }
        expect(apiserver.live.delete(coordKey)).toBe(true)

        apiserver.api.createNamespacedNetworkPolicy.mockClear()
        vi.setSystemTime(SECOND_PASS_AT)
        await reconciler.refreshRuntimeHttpEgressNetworkPolicies(
          'sandbox-recipes',
          'test-wf',
          'uid-123',
          spec
        )

        // The unchanged sibling must not lend its old resolved-at to a policy
        // that did not exist: the recreated one carries the current time.
        const created = apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
        expect(created.map(arg => arg.body.metadata?.name)).toEqual(['test-wf-coord-to-wrc'])
        expect(created[0]!.body.metadata?.annotations?.[RESOLVED_AT]).toBe(
          SECOND_PASS_AT.toISOString()
        )
        expect(apiserver.live.get(coordKey)?.metadata?.annotations?.[RESOLVED_AT]).toBe(
          SECOND_PASS_AT.toISOString()
        )
      })

      it('stamps the current time when the resolved set changes', async () => {
        const resolve = vi
          .fn()
          .mockResolvedValueOnce(['93.184.216.34/32'])
          .mockResolvedValueOnce(['93.184.216.35/32'])
        const { api } = await refreshTwice(resolve)

        const replaced = api.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
        expect(replaced.map(arg => arg.name).sort()).toEqual(RUNTIME_EGRESS_POLICIES)
        for (const arg of replaced) {
          expect(arg.body.metadata?.annotations).toMatchObject({
            [RESOLVED_AT]: SECOND_PASS_AT.toISOString(),
            'clerum.io/runtime-http-egress-current-cidrs': '93.184.216.35/32',
            'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
          })
        }
      })

      it('keeps the live resolved-at through a DNS-failure prune, so later failures and a healthy refresh write nothing', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const apiserver = makeApiserverNetworkingApi()
          const resolve = vi
            .fn()
            .mockResolvedValueOnce(['93.184.216.34/32'])
            .mockResolvedValueOnce(['93.184.216.35/32'])
            .mockRejectedValueOnce(new Error('ENOTFOUND'))
            .mockRejectedValueOnce(new Error('ENOTFOUND'))
            .mockResolvedValueOnce(['93.184.216.35/32'])
          const reconciler = new WorkflowReconciler(
            makeDeps({
              networkingApi: apiserver.api as never,
              resolveRuntimeHttpEgressCidrs: resolve,
              config: { ...makeConfig(), enableSnippetRuntime: true } as never,
            })
          )
          const spec = runtimeEgressSpec()
          const refresh = () =>
            reconciler.refreshRuntimeHttpEgressNetworkPolicies(
              'sandbox-recipes',
              'test-wf',
              'uid-123',
              spec
            )
          const clearCalls = () => {
            apiserver.api.createNamespacedNetworkPolicy.mockClear()
            apiserver.api.readNamespacedNetworkPolicy.mockClear()
            apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
          }
          const liveAnnotations = (name: string) =>
            apiserver.live.get(apiserver.key('sandbox-recipes', name))?.metadata?.annotations

          // Two healthy passes leave the live policy with a resolved-at and a
          // previous CIDR whose overlap window is still open.
          vi.useFakeTimers({ toFake: ['Date'] })
          vi.setSystemTime(FIRST_PASS_AT)
          await refresh()
          vi.setSystemTime(SECOND_PASS_AT)
          await refresh()
          for (const name of RUNTIME_EGRESS_POLICIES) {
            expect(liveAnnotations(name)).toMatchObject({
              [RESOLVED_AT]: SECOND_PASS_AT.toISOString(),
              'clerum.io/runtime-http-egress-previous-cidrs': '93.184.216.34/32',
            })
          }

          // DNS fails after the overlap expired: the prune drops the expired
          // CIDR in one PUT per policy and carries the live resolved-at over.
          const pruneAt = new Date(SECOND_PASS_AT.getTime() + 24 * 60 * 60 * 1000)
          clearCalls()
          vi.setSystemTime(pruneAt)
          await expect(refresh()).rejects.toThrow('ENOTFOUND')
          const pruned = apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg)
          expect(pruned.map(arg => arg.name).sort()).toEqual(RUNTIME_EGRESS_POLICIES)
          for (const arg of pruned) {
            expect(arg.body.metadata?.annotations?.[RESOLVED_AT]).toBe(SECOND_PASS_AT.toISOString())
            expect(
              arg.body.metadata?.annotations?.['clerum.io/runtime-http-egress-previous-cidrs']
            ).toBeUndefined()
          }

          // A second DNS failure reads the policies and has nothing to prune.
          clearCalls()
          vi.setSystemTime(new Date(pruneAt.getTime() + 30_000))
          await expect(refresh()).rejects.toThrow('ENOTFOUND')
          expect(readPolicyNames(apiserver.api)).toEqual(
            expect.arrayContaining(RUNTIME_EGRESS_POLICIES)
          )
          expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)

          // DNS recovers with the same set: the refresh finds the policies
          // converged and keeps the resolved-at the prune preserved.
          clearCalls()
          vi.setSystemTime(new Date(pruneAt.getTime() + 60_000))
          await expect(refresh()).resolves.toEqual({ conflicts: [], retryPending: false })
          expect(resolve).toHaveBeenCalledTimes(5)
          expect(readPolicyNames(apiserver.api)).toEqual(
            expect.arrayContaining(RUNTIME_EGRESS_POLICIES)
          )
          expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          for (const name of RUNTIME_EGRESS_POLICIES) {
            expect(liveAnnotations(name)?.[RESOLVED_AT]).toBe(SECOND_PASS_AT.toISOString())
            expect(
              logs.entries.filter(entry =>
                String(entry.msg).includes(`Unchanged NetworkPolicy "${name}"`)
              )
            ).toHaveLength(1)
          }
        } finally {
          logs.restore()
        }
      })

      describe('DNS rollover while the snippet runner policy is left unwritten', () => {
        const COORD = 'test-wf-coord-to-wrc'
        const RUNNER = 'test-wf-snippet-runner-egress'
        const OLD_CIDR = '93.184.216.34/32'
        const NEW_CIDR = '93.184.216.35/32'
        const CURRENT_CIDRS = 'clerum.io/runtime-http-egress-current-cidrs'
        const PREVIOUS_CIDRS = 'clerum.io/runtime-http-egress-previous-cidrs'
        const REFRESH_INTERVAL_MS = 30_000
        // 40 minutes of refreshes: eight times the 300 s overlap window.
        const REFRESHES_AFTER_ROLLOVER = 80

        /**
         * Create both policies on the old CIDR, freeze the runner policy, move
         * DNS to the new CIDR and refresh every 30 s. Returns the coordinator
         * replaces seen during the loop; the mocks and `logs` then hold only one
         * further refresh.
         */
        async function rolloverWithFrozenRunner(
          logs: ReturnType<typeof captureRunLaneNetworkPolicyLogs>,
          freeze: (runner: k8s.V1NetworkPolicy) => void
        ) {
          const apiserver = makeApiserverNetworkingApi()
          let resolved = [OLD_CIDR]
          const resolve = vi.fn(async () => resolved)
          const reconciler = new WorkflowReconciler(
            makeDeps({
              networkingApi: apiserver.api as never,
              resolveRuntimeHttpEgressCidrs: resolve,
              config: { ...makeConfig(), enableSnippetRuntime: true } as never,
            })
          )
          // A custom coordinator adds coord-to-wrc next to snippet-runner-egress.
          const spec: WorkflowRecipeSpec = {
            ...runtimeEgressSpec(),
            coordinatorImage: 'registry.example/custom-coordinator:1',
          }
          const refresh = () =>
            reconciler.refreshRuntimeHttpEgressNetworkPolicies(
              'sandbox-recipes',
              'test-wf',
              'uid-123',
              spec
            )
          const coordKey = apiserver.key('sandbox-recipes', COORD)
          const runnerKey = apiserver.key('sandbox-recipes', RUNNER)

          vi.useFakeTimers({ toFake: ['Date'] })
          let now = FIRST_PASS_AT.getTime()
          vi.setSystemTime(now)
          await refresh()
          for (const policyKey of [coordKey, runnerKey]) {
            expect(apiserver.live.get(policyKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(
              OLD_CIDR
            )
          }
          freeze(apiserver.live.get(runnerKey)!)

          resolved = [NEW_CIDR]
          apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
          for (let pass = 0; pass < REFRESHES_AFTER_ROLLOVER; pass += 1) {
            now += REFRESH_INTERVAL_MS
            vi.setSystemTime(now)
            await refresh()
          }
          const coordReplacesDuringLoop = apiserver.api.replaceNamespacedNetworkPolicy.mock.calls
            .map(([arg]) => arg)
            .filter(arg => arg.name === COORD)

          apiserver.api.createNamespacedNetworkPolicy.mockClear()
          apiserver.api.readNamespacedNetworkPolicy.mockClear()
          apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
          logs.entries.length = 0
          now += REFRESH_INTERVAL_MS
          vi.setSystemTime(now)
          const last = await refresh()
          return { ...apiserver, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last }
        }

        it('drops the old CIDR from the owned policy and stops replacing it when the sibling has a foreign owner', async () => {
          const logs = captureRunLaneNetworkPolicyLogs()
          try {
            const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =
              await rolloverWithFrozenRunner(logs, runner => {
                runner.metadata = {
                  ...runner.metadata,
                  ownerReferences: [
                    {
                      apiVersion: 'apps/v1',
                      kind: 'Deployment',
                      name: 'another-controller',
                      uid: 'foreign-owner-uid',
                      controller: true,
                    },
                  ],
                }
              })

            // Liveness: every pass resolved DNS; the last one read both policies
            // by name for the egress state and again for the apply, and reached
            // the ownership branch for the runner.
            expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)
            expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])
            expect(last).toEqual({
              conflicts: [{ policy: RUNNER, reason: 'owner-reference-mismatch' }],
              retryPending: false,
            })
            expect(
              logs.entries.some(
                entry =>
                  entry.level === 'warn' &&
                  entry.reason === 'owner-reference-mismatch' &&
                  String(entry.msg).includes(RUNNER)
              )
            ).toBe(true)
            // The foreign policy keeps the old CIDR it was frozen with.
            expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)

            // One replace opens the overlap, one closes it once it expires.
            expect(coordReplacesDuringLoop).toHaveLength(2)
            const lastReplaced = coordReplacesDuringLoop.at(-1)!.body
            expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])
            expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()
            expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])

            expect(
              api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)
            ).toHaveLength(0)
            expect(
              logs.entries.filter(entry =>
                String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)
              )
            ).toHaveLength(1)
          } finally {
            logs.restore()
          }
        })

        it('drops the old CIDR from the owned policy and stops replacing it when the sibling is terminating', async () => {
          const logs = captureRunLaneNetworkPolicyLogs()
          try {
            const { api, live, resolve, coordKey, runnerKey, coordReplacesDuringLoop, last } =
              await rolloverWithFrozenRunner(logs, runner => {
                runner.metadata = {
                  ...runner.metadata,
                  deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
                }
              })

            expect(resolve).toHaveBeenCalledTimes(REFRESHES_AFTER_ROLLOVER + 2)
            expect(readPolicyNames(api)).toEqual([COORD, COORD, RUNNER, RUNNER])
            expect(last).toEqual({ conflicts: [], retryPending: true })
            expect(
              logs.entries.some(
                entry =>
                  entry.level === 'warn' &&
                  entry.reason === 'terminating' &&
                  String(entry.msg).includes(`NetworkPolicy "${RUNNER}"`)
              )
            ).toBe(true)
            expect(live.get(runnerKey)?.metadata?.annotations?.[CURRENT_CIDRS]).toBe(OLD_CIDR)

            expect(coordReplacesDuringLoop).toHaveLength(2)
            const lastReplaced = coordReplacesDuringLoop.at(-1)!.body
            expect(publicHttpEgressCidrs(lastReplaced)).toEqual([NEW_CIDR])
            expect(lastReplaced.metadata?.annotations?.[PREVIOUS_CIDRS]).toBeUndefined()
            expect(publicHttpEgressCidrs(live.get(coordKey)!)).toEqual([NEW_CIDR])

            expect(
              api.replaceNamespacedNetworkPolicy.mock.calls.filter(([arg]) => arg.name === COORD)
            ).toHaveLength(0)
            expect(
              logs.entries.filter(entry =>
                String(entry.msg).includes(`Unchanged NetworkPolicy "${COORD}"`)
              )
            ).toHaveLength(1)
          } finally {
            logs.restore()
          }
        })
      })
    })

    it('reports a foreign-owned run-lane policy as a condition and keeps the run alive', async () => {
      const { api, live, key } = makeApiserverNetworkingApi()
      const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
      const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

      const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
      expect(first.workflowPhase).not.toBe('failed')
      const foreign = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))
      expect(foreign).toBeDefined()
      // Drift the spec too, so that without the ownership check the pass would PUT.
      foreign!.spec = { ...foreign!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
      foreign!.metadata = {
        ...foreign!.metadata,
        ownerReferences: [
          {
            apiVersion: 'apps/v1',
            kind: 'Deployment',
            name: 'another-controller',
            uid: 'foreign-owner-uid',
            controller: true,
          },
        ],
      }
      api.readNamespacedNetworkPolicy.mockClear()
      api.replaceNamespacedNetworkPolicy.mockClear()

      const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

      const expectedNames = RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
      expect(readPolicyNames(api)).toEqual(expectedNames)
      expect(second.phase).not.toBe('failed')
      expect(second.workflowPhase).not.toBe('failed')
      expect(second.networkPolicyOwnershipConditions).toEqual([
        expect.objectContaining({
          type: 'WorkflowNetworkPolicyOwnership',
          status: 'False',
          reason: 'OwnershipConflict',
          message: expect.stringContaining('test-wf-coord-to-wrc'),
        }),
      ])
      expect(
        api.replaceNamespacedNetworkPolicy.mock.calls.filter(
          ([arg]) => arg.name === 'test-wf-coord-to-wrc'
        )
      ).toHaveLength(0)
      // A conflict is not transient: it waits for an operator, not a timer.
      expect(second.networkPolicyRetryPending).toBeFalsy()
    })

    it('flags a retry for a run-lane policy that is being deleted and leaves it unwritten', async () => {
      const logs = captureRunLaneNetworkPolicyLogs()
      try {
        const { api, live, key } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        expect(first.workflowPhase).not.toBe('failed')
        expect(first.networkPolicyRetryPending).toBeFalsy()
        const terminating = live.get(key('sandbox-recipes', 'test-wf-coord-to-wrc'))
        expect(terminating).toBeDefined()
        // Drift the spec too, so that without the deletion check the pass would PUT.
        terminating!.spec = {
          ...terminating!.spec!,
          podSelector: { matchLabels: { drifted: 'yes' } },
        }
        terminating!.metadata = {
          ...terminating!.metadata,
          deletionTimestamp: new Date('2026-09-23T10:00:00Z'),
        }
        api.readNamespacedNetworkPolicy.mockClear()
        api.replaceNamespacedNetworkPolicy.mockClear()
        logs.entries.length = 0

        const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(readPolicyNames(api)).toContain('test-wf-coord-to-wrc')
        expect(second.networkPolicyRetryPending).toBe(true)
        expect(second.phase).not.toBe('failed')
        expect(second.workflowPhase).not.toBe('failed')
        // A policy being deleted is not a conflict: the pass reached the apply
        // and found none, so the owned set holds only the RetryPending marker.
        expect(second.networkPolicyOwnershipConditions).toEqual([
          expect.objectContaining({
            type: 'WorkflowNetworkPoliciesConverged',
            status: 'False',
            reason: 'RetryPending',
          }),
        ])
        expect(
          api.replaceNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.name === 'test-wf-coord-to-wrc'
          )
        ).toHaveLength(0)
        // The retry warn carries the decision's reason, not a fixed phrase.
        expect(
          logs.entries.filter(
            entry =>
              entry.level === 'warn' &&
              String(entry.msg).includes('test-wf-coord-to-wrc') &&
              String(entry.msg).includes('retrying later')
          )
        ).toEqual([expect.objectContaining({ reason: 'terminating' })])
      } finally {
        logs.restore()
      }
    })

    it('flags a contended retry, without failing, after a concurrent writer wins all three replaces of a run-lane policy', async () => {
      const logs = captureRunLaneNetworkPolicyLogs()
      try {
        const { api, live, key } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        expect(first.workflowPhase).not.toBe('failed')
        const contendedKey = key('sandbox-recipes', 'test-wf-coord-to-wrc')
        const contended = live.get(contendedKey)
        expect(contended).toBeDefined()
        contended!.spec = { ...contended!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }
        const liveResourceVersion = contended!.metadata!.resourceVersion

        // Another writer updates the policy just before each of our replaces, so
        // every one carries a resourceVersion that is already stale.
        const apiserverReplace = api.replaceNamespacedNetworkPolicy.getMockImplementation()!
        let concurrentWrites = 0
        api.replaceNamespacedNetworkPolicy.mockImplementation(async arg => {
          if (arg.name === 'test-wf-coord-to-wrc') {
            concurrentWrites += 1
            const current = live.get(contendedKey)!
            live.set(contendedKey, {
              ...current,
              metadata: { ...current.metadata, resourceVersion: `concurrent-${concurrentWrites}` },
            })
          }
          return apiserverReplace(arg)
        })
        api.readNamespacedNetworkPolicy.mockClear()
        api.replaceNamespacedNetworkPolicy.mockClear()
        logs.entries.length = 0

        const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(
          api.readNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.name === 'test-wf-coord-to-wrc'
          )
        ).toHaveLength(3)
        // Bounded: three writes, the same ceiling as the create-then-two-replaces
        // apply this PR replaces.
        expect(
          api.replaceNamespacedNetworkPolicy.mock.calls
            .filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
            .map(([arg]) => arg.body.metadata?.resourceVersion)
        ).toEqual([liveResourceVersion, 'concurrent-1', 'concurrent-2'])
        expect(second.phase).not.toBe('failed')
        expect(second.workflowPhase).not.toBe('failed')
        expect(second.networkPolicyRetryPending).toBe(true)
        expect(
          logs.entries.filter(
            entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
          )
        ).toEqual([
          expect.objectContaining({
            reason: 'contended',
            msg: 'NetworkPolicy "test-wf-coord-to-wrc" is still contended after 3 write attempts; retrying later',
          }),
        ])
      } finally {
        logs.restore()
      }
    })

    describe('replace failures after the read', () => {
      async function reconcileWithFailingReplace(
        failReplace: (
          arg: { name: string; namespace: string },
          live: Map<string, k8s.V1NetworkPolicy>,
          api: ReturnType<typeof makeApiserverNetworkingApi>['api']
        ) => Promise<never> | undefined,
        logs?: { entries: Array<Record<string, unknown>> }
      ) {
        const apiserver = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(
          makeDeps({ networkingApi: apiserver.api as never })
        )
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        expect(first.workflowPhase).not.toBe('failed')
        const drifted = apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
        expect(drifted).toBeDefined()
        // What the first pass stored: the desired policy as the apiserver keeps it.
        const convergedSpec = structuredClone(drifted!.spec!)
        drifted!.spec = { ...drifted!.spec!, podSelector: { matchLabels: { drifted: 'yes' } } }

        const apiserverReplace =
          apiserver.api.replaceNamespacedNetworkPolicy.getMockImplementation()!
        apiserver.api.replaceNamespacedNetworkPolicy.mockImplementation(
          async arg => failReplace(arg, apiserver.live, apiserver.api) ?? apiserverReplace(arg)
        )
        apiserver.api.readNamespacedNetworkPolicy.mockClear()
        apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
        apiserver.api.createNamespacedNetworkPolicy.mockClear()
        // Only the second pass is under test; drop the first pass's "Created" lines.
        logs?.entries.splice(0)

        const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        const contendedCalls = (mock: typeof apiserver.api.readNamespacedNetworkPolicy) =>
          mock.mock.calls.filter(([arg]) => arg.name === 'test-wf-coord-to-wrc')
        return {
          second,
          apiserver,
          convergedSpec,
          reads: contendedCalls(apiserver.api.readNamespacedNetworkPolicy),
          replaces: apiserver.api.replaceNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.name === 'test-wf-coord-to-wrc'
          ),
          creates: apiserver.api.createNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.body.metadata?.name === 'test-wf-coord-to-wrc'
          ),
        }
      }

      it('recreates the policy in the same pass when it is deleted between the read and the replace', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { second, apiserver, convergedSpec, reads, replaces, creates } =
            await reconcileWithFailingReplace((arg, live) => {
              if (arg.name !== 'test-wf-coord-to-wrc') return undefined
              // Another actor deletes the policy after our read, so the
              // apiserver double answers the PUT with 404.
              expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
              return undefined
            }, logs)

          // Read, 404 on the PUT, read again (absent), create.
          expect(reads).toHaveLength(2)
          expect(replaces).toHaveLength(1)
          expect(creates).toHaveLength(1)
          // The recreated object is the desired policy, nothing wider.
          expect(
            apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
          ).toEqual(convergedSpec)
          expect(second.phase).not.toBe('failed')
          expect(second.workflowPhase).not.toBe('failed')
          expect(second.networkPolicyRetryPending).toBeFalsy()
          expect(
            logs.entries
              .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
              .map(entry => [entry.level, entry.msg])
          ).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
        } finally {
          logs.restore()
        }
      })

      it('propagates a replace error that is neither a conflict nor a deletion', async () => {
        const { second, reads, replaces } = await reconcileWithFailingReplace(arg => {
          if (arg.name !== 'test-wf-coord-to-wrc') return undefined
          return Promise.reject(
            Object.assign(new Error('HTTP-Code: 422 Message: NetworkPolicy is invalid'), {
              code: 422,
            })
          )
        })

        expect(reads).toHaveLength(1)
        expect(replaces).toHaveLength(1)
        expect(second.workflowPhase).toBe('failed')
        expect(second.message).toContain('HTTP-Code: 422')
      })

      it('recreates the policy in the same pass when it is deleted after a replace conflict', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { second, apiserver, convergedSpec, reads, replaces, creates } =
            await reconcileWithFailingReplace((arg, live) => {
              if (arg.name !== 'test-wf-coord-to-wrc') return undefined
              // Another actor deletes the policy and our PUT loses the race, so
              // the re-read after the conflict finds nothing.
              expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
              return Promise.reject({ code: 409 })
            }, logs)

          expect(reads).toHaveLength(2)
          expect(replaces).toHaveLength(1)
          expect(creates).toHaveLength(1)
          expect(
            apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
          ).toEqual(convergedSpec)
          expect(second.phase).not.toBe('failed')
          expect(second.workflowPhase).not.toBe('failed')
          expect(second.networkPolicyRetryPending).toBeFalsy()
          expect(
            logs.entries
              .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
              .map(entry => [entry.level, entry.msg])
          ).toEqual([['info', 'Created NetworkPolicy "test-wf-coord-to-wrc"']])
        } finally {
          logs.restore()
        }
      })

      it('leaves an equal copy alone when another writer recreates the policy between our 404 and our create', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          let createRaces = 0
          const { second, apiserver, convergedSpec, reads, replaces, creates } =
            await reconcileWithFailingReplace((arg, live, api) => {
              if (arg.name !== 'test-wf-coord-to-wrc') return undefined
              expect(live.delete(`${arg.namespace}/${arg.name}`)).toBe(true)
              // Before our create lands, another writer stores the desired
              // policy, so our POST answers 409 and the next read finds it.
              const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
              api.createNamespacedNetworkPolicy.mockImplementationOnce(async create => {
                createRaces += 1
                await apiserverCreate(create)
                return apiserverCreate(create)
              })
              return undefined
            }, logs)

          expect(createRaces).toBe(1)
          // Read, 404 on the PUT, read (absent), create 409, read (equal).
          expect(reads).toHaveLength(3)
          expect(replaces).toHaveLength(1)
          expect(creates).toHaveLength(1)
          expect(
            apiserver.live.get(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))!.spec
          ).toEqual(convergedSpec)
          expect(second.workflowPhase).not.toBe('failed')
          expect(second.networkPolicyRetryPending).toBeFalsy()
          expect(
            logs.entries
              .filter(entry => String(entry.msg).includes('test-wf-coord-to-wrc'))
              .map(entry => [entry.level, entry.msg])
          ).toEqual([['info', 'Unchanged NetworkPolicy "test-wf-coord-to-wrc"; skipping update']])
        } finally {
          logs.restore()
        }
      })

      // The decision is taken again on the object the re-read returns, not
      // reused from the first read: a writer that wins the replace may also
      // take the policy over or start deleting it.
      function concurrentWriterOnFirstReplace(
        rewrite: (current: k8s.V1NetworkPolicy) => k8s.V1NetworkPolicy
      ) {
        let concurrentWrites = 0
        return (
          arg: { name: string; namespace: string },
          live: Map<string, k8s.V1NetworkPolicy>
        ): Promise<never> | undefined => {
          if (arg.name !== 'test-wf-coord-to-wrc' || concurrentWrites > 0) return undefined
          concurrentWrites += 1
          const liveKey = `${arg.namespace}/${arg.name}`
          const current = live.get(liveKey)
          expect(current).toBeDefined()
          const rewritten = rewrite(current!)
          live.set(liveKey, {
            ...rewritten,
            metadata: { ...rewritten.metadata, resourceVersion: 'concurrent-1' },
          })
          return Promise.reject({ code: 409 })
        }
      }

      it('reports a conflict, without a second replace, when the re-read after a replace conflict finds a foreign owner', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { second, reads, replaces } = await reconcileWithFailingReplace(
            concurrentWriterOnFirstReplace(current => ({
              ...current,
              metadata: {
                ...current.metadata,
                ownerReferences: [
                  {
                    apiVersion: 'apps/v1',
                    kind: 'Deployment',
                    name: 'another-controller',
                    uid: 'foreign-owner-uid',
                    controller: true,
                  },
                ],
              },
            }))
          )

          expect(reads).toHaveLength(2)
          expect(replaces).toHaveLength(1)
          expect(second.phase).not.toBe('failed')
          expect(second.workflowPhase).not.toBe('failed')
          expect(second.networkPolicyOwnershipConditions).toEqual([
            expect.objectContaining({
              type: 'WorkflowNetworkPolicyOwnership',
              status: 'False',
              reason: 'OwnershipConflict',
              message:
                'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
            }),
          ])
          // A conflict waits for an operator, not a timer.
          expect(second.networkPolicyRetryPending).toBeFalsy()
          expect(
            logs.entries.filter(
              entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
            )
          ).toEqual([
            expect.objectContaining({
              reason: 'owner-reference-mismatch',
              msg: 'NetworkPolicy "test-wf-coord-to-wrc" is not owned by this workflow; leaving it',
            }),
          ])
        } finally {
          logs.restore()
        }
      })

      it('flags a retry, without a second replace, when the re-read after a replace conflict finds the policy terminating', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { second, reads, replaces } = await reconcileWithFailingReplace(
            concurrentWriterOnFirstReplace(current => ({
              ...current,
              metadata: {
                ...current.metadata,
                deletionTimestamp: new Date('2026-09-23T10:00:00.000Z'),
              },
            }))
          )

          expect(reads).toHaveLength(2)
          expect(replaces).toHaveLength(1)
          expect(second.phase).not.toBe('failed')
          expect(second.workflowPhase).not.toBe('failed')
          expect(second.networkPolicyRetryPending).toBe(true)
          // Being deleted is not a conflict: the owned set holds only the
          // RetryPending marker.
          expect(second.networkPolicyOwnershipConditions).toEqual([
            expect.objectContaining({
              type: 'WorkflowNetworkPoliciesConverged',
              status: 'False',
              reason: 'RetryPending',
            }),
          ])
          expect(
            logs.entries.filter(
              entry => entry.level === 'warn' && String(entry.msg).includes('test-wf-coord-to-wrc')
            )
          ).toEqual([
            expect.objectContaining({
              reason: 'terminating',
              msg: 'NetworkPolicy "test-wf-coord-to-wrc" cannot be written yet; retrying later',
            }),
          ])
        } finally {
          logs.restore()
        }
      })
    })

    describe('create conflicts, read errors and stale runtime state', () => {
      const CONTENDED = 'test-wf-coord-to-wrc'

      function callsFor<T extends { name?: string; body?: k8s.V1NetworkPolicy }>(mock: {
        mock: { calls: Array<[T]> }
      }): T[] {
        return mock.mock.calls
          .map(([arg]) => arg)
          .filter(arg => (arg.name ?? arg.body?.metadata?.name) === CONTENDED)
      }

      it('replaces the winner of a create race with the winner resourceVersion', async () => {
        const { api, live, key } = makeApiserverNetworkingApi()
        const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
        let winnerResourceVersion: string | undefined
        api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
          if (arg.body.metadata?.name === CONTENDED) {
            // Another writer creates a drifted copy between our 404 read and our POST.
            const winner = await apiserverCreate({
              namespace: arg.namespace,
              body: {
                ...arg.body,
                spec: { ...arg.body.spec!, podSelector: { matchLabels: { app: 'race-winner' } } },
              },
            })
            winnerResourceVersion = winner.metadata?.resourceVersion
          }
          return apiserverCreate(arg)
        })
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(result.workflowPhase).not.toBe('failed')
        expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
        expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
        const replaces = callsFor(api.replaceNamespacedNetworkPolicy)
        expect(replaces).toHaveLength(1)
        expect(winnerResourceVersion).toBeDefined()
        expect(replaces[0]!.body.metadata?.resourceVersion).toBe(winnerResourceVersion)
        expect(live.get(key('sandbox-recipes', CONTENDED))!.spec!.podSelector).not.toEqual({
          matchLabels: { app: 'race-winner' },
        })
      })

      it('creates again in the same pass when a create conflicts and the re-read finds nothing', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { api, live, key } = makeApiserverNetworkingApi()
          const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
          let conflicts = 0
          api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
            // The winner of the first race is deleted again before our re-read.
            if (arg.body.metadata?.name === CONTENDED && conflicts === 0) {
              conflicts += 1
              throw { code: 409 }
            }
            return apiserverCreate(arg)
          })
          const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
          const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

          const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

          expect(conflicts).toBe(1)
          expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(2)
          expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(2)
          expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
          expect(live.has(key('sandbox-recipes', CONTENDED))).toBe(true)
          expect(result.phase).not.toBe('failed')
          expect(result.workflowPhase).not.toBe('failed')
          expect(result.networkPolicyRetryPending).toBeFalsy()
          expect(
            logs.entries
              .filter(entry => String(entry.msg).includes(CONTENDED))
              .map(entry => [entry.level, entry.msg])
          ).toEqual([['info', `Created NetworkPolicy "${CONTENDED}"`]])
        } finally {
          logs.restore()
        }
      })

      it('flags a contended retry after three creates that all conflict, and writes no fourth time', async () => {
        const logs = captureRunLaneNetworkPolicyLogs()
        try {
          const { api } = makeApiserverNetworkingApi()
          const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
          api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
            // Every winner is deleted again before our re-read.
            if (arg.body.metadata?.name === CONTENDED) throw { code: 409 }
            return apiserverCreate(arg)
          })
          const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
          const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

          const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

          expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(3)
          expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(3)
          expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
          expect(result.phase).not.toBe('failed')
          expect(result.workflowPhase).not.toBe('failed')
          expect(result.networkPolicyRetryPending).toBe(true)
          expect(
            logs.entries.filter(
              entry => entry.level === 'warn' && String(entry.msg).includes(CONTENDED)
            )
          ).toEqual([
            expect.objectContaining({
              reason: 'contended',
              msg: `NetworkPolicy "${CONTENDED}" is still contended after 3 write attempts; retrying later`,
            }),
          ])
        } finally {
          logs.restore()
        }
      })

      it('fails the run on a create error other than 409 and does not replace', async () => {
        const { api } = makeApiserverNetworkingApi()
        const apiserverCreate = api.createNamespacedNetworkPolicy.getMockImplementation()!
        api.createNamespacedNetworkPolicy.mockImplementation(async arg => {
          if (arg.body.metadata?.name === CONTENDED) {
            throw Object.assign(new Error('HTTP-Code: 500 Message: etcd unavailable'), {
              code: 500,
            })
          }
          return apiserverCreate(arg)
        })
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)
        expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(1)
        expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
        // A 500 is not a conflict: it leaves the apply and reaches the reconcile catch.
        expect(result.message).toContain('HTTP-Code: 500')
      })

      it('fails the run on a read error other than 404 and writes nothing for that policy', async () => {
        const { api } = makeApiserverNetworkingApi()
        const apiserverRead = api.readNamespacedNetworkPolicy.getMockImplementation()!
        api.readNamespacedNetworkPolicy.mockImplementation(async arg => {
          if (arg.name === CONTENDED) {
            throw Object.assign(new Error('HTTP-Code: 403 Message: forbidden'), { code: 403 })
          }
          return apiserverRead(arg)
        })
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

        const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        // A 403 is not absence: creating over it would hide a permission defect.
        expect(callsFor(api.readNamespacedNetworkPolicy)).toHaveLength(1)
        expect(callsFor(api.createNamespacedNetworkPolicy)).toHaveLength(0)
        expect(callsFor(api.replaceNamespacedNetworkPolicy)).toHaveLength(0)
        expect(result.workflowPhase).toBe('failed')
        expect(result.message).toContain('HTTP-Code: 403')
      })

      it('replaces a policy whose only drift is a stale runtime HTTP egress annotation', async () => {
        const { api, live, key } = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(makeDeps({ networkingApi: api as never }))
        const spec = makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })
        const STALE = 'clerum.io/runtime-http-egress-previous-cidrs'

        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)
        expect(first.workflowPhase).not.toBe('failed')
        const stale = live.get(key('sandbox-recipes', CONTENDED))
        expect(stale).toBeDefined()
        expect(stale!.metadata?.annotations?.[STALE]).toBeUndefined()
        stale!.metadata = {
          ...stale!.metadata,
          annotations: { ...stale!.metadata?.annotations, [STALE]: '198.51.100.7/32' },
        }
        api.replaceNamespacedNetworkPolicy.mockClear()

        const second = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec)

        expect(second.workflowPhase).not.toBe('failed')
        const replaces = callsFor(api.replaceNamespacedNetworkPolicy)
        expect(replaces).toHaveLength(1)
        expect(replaces[0]!.body.metadata?.annotations?.[STALE]).toBeUndefined()
        expect(live.get(key('sandbox-recipes', CONTENDED))!.metadata?.annotations?.[STALE]).toBe(
          undefined
        )
      })
    })

    // The recipe reconciler calls this from the running and active
    // short-circuits, which never reach reconcile(), to retry a policy a
    // previous pass left pending.
    describe('retryRunLaneNetworkPolicies', () => {
      const spec = () =>
        makeSpec({ agent: undefined, steps: [{ id: 'prepare', run: snippetRun() }] })

      async function convergedRunLane() {
        const apiserver = makeApiserverNetworkingApi()
        const reconciler = new WorkflowReconciler(
          makeDeps({ networkingApi: apiserver.api as never })
        )
        const first = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
        expect(first.workflowPhase).not.toBe('failed')
        expect([...apiserver.live.keys()].sort()).toEqual(RUN_LANE_POLICY_NAMES)
        // reconcile() LISTs leftover run-lane policies and still DELETEs the
        // legacy mcp-servers-egress-internet once (B3(c)). A zero delete count
        // below means the retry skipped those paths, not that no prune exists.
        expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalled()
        apiserver.api.createNamespacedNetworkPolicy.mockClear()
        apiserver.api.readNamespacedNetworkPolicy.mockClear()
        apiserver.api.replaceNamespacedNetworkPolicy.mockClear()
        apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
        return { apiserver, reconciler }
      }

      it('recreates an absent run-lane policy and deletes nothing', async () => {
        const { apiserver, reconciler } = await convergedRunLane()
        const absentKey = apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
        const convergedSpec = structuredClone(apiserver.live.get(absentKey)!.spec)
        expect(apiserver.live.delete(absentKey)).toBe(true)

        const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())

        expect(summary).toEqual({ conflicts: [], retryPending: false })
        expect(readPolicyNames(apiserver.api)).toEqual(
          RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
        )
        expect(
          apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
            ([arg]) => arg.body.metadata?.name
          )
        ).toEqual(['test-wf-coord-to-wrc'])
        expect(apiserver.live.get(absentKey)!.spec).toEqual(convergedSpec)
        expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
      })

      it('writes nothing when every run-lane policy is live and equal', async () => {
        const { apiserver, reconciler } = await convergedRunLane()

        const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())

        expect(summary).toEqual({ conflicts: [], retryPending: false })
        expect(readPolicyNames(apiserver.api)).toEqual(
          RUN_LANE_POLICY_NAMES.map(entry => entry.split('/')[1]!).sort()
        )
        expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
      })

      it('still reports the retry, and writes nothing, while a policy is being deleted', async () => {
        const { apiserver, reconciler } = await convergedRunLane()
        const terminating = apiserver.live.get(
          apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc')
        )!
        terminating.spec = {
          ...terminating.spec!,
          podSelector: { matchLabels: { drifted: 'yes' } },
        }
        terminating.metadata = {
          ...terminating.metadata,
          deletionTimestamp: new Date('2026-09-24T10:00:00Z'),
        }

        const summary = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())

        expect(summary).toEqual({ conflicts: [], retryPending: true })
        expect(readPolicyNames(apiserver.api)).toContain('test-wf-coord-to-wrc')
        expect(apiserver.api.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(apiserver.api.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
      })

      // The retry runs only from the in-progress and active short-circuits,
      // where the run's pods are live and may still use any lane the spec
      // wanted when they were created. It applies what the current spec wants
      // and prunes nothing; the next reconcile() pass prunes the rest.
      describe('when the spec no longer wants a live lane', () => {
        const SNIPPET_LANES = [
          'test-wf-coord-to-snippet-runner',
          'test-wf-coord-to-snippet-runner-ingress',
          'test-wf-snippet-runner-egress',
        ]
        const AGENT_LANE = 'test-wf-coord-to-mcp-host'

        it('keeps it on a mid-run retry and still applies the lanes the spec wants', async () => {
          const { apiserver, reconciler } = await convergedRunLane()
          apiserver.api.listNamespacedNetworkPolicy.mockClear()

          // An agent recipe with a run id wants the mcp-host lanes and none of
          // the snippet lanes the converged snippet recipe created.
          const summary = await reconciler.retryRunLaneNetworkPolicies(
            'test-wf',
            'uid-123',
            makeSpec(),
            'test-wf',
            'run-1'
          )

          // Liveness witness: the retry built the agent lane and created it.
          expect(
            apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
              ([arg]) => arg.body.metadata?.name
            )
          ).toContain(AGENT_LANE)
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
          for (const name of SNIPPET_LANES) {
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(true)
          }
          expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(summary).toEqual({ conflicts: [], retryPending: false })
        })

        it('prunes it on the next reconcile() pass', async () => {
          const { apiserver, reconciler } = await convergedRunLane()
          await reconciler.retryRunLaneNetworkPolicies(
            'test-wf',
            'uid-123',
            makeSpec(),
            'test-wf',
            'run-1'
          )
          expect(
            apiserver.live.has(apiserver.key('sandbox-recipes', 'test-wf-snippet-runner-egress'))
          ).toBe(true)

          const result = await reconciler.reconcile(
            'test-wf',
            'uid-123',
            'sandbox-recipes',
            makeSpec(),
            undefined,
            undefined,
            'test-wf',
            'run-1'
          )

          expect(result.workflowPhase).not.toBe('failed')
          const deleted = apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(
            ([arg]) => arg.name
          )
          for (const name of SNIPPET_LANES) {
            expect(deleted, name).toContain(name)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', name)), name).toBe(false)
          }
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', AGENT_LANE))).toBe(true)
        })
      })

      // A reconcile() prune that failed publishes PrunePending with no retry
      // flag. The retry does not prune and carries no prune fact, so the
      // marker stays until the next reconcile() pass prunes the leftover.
      describe('when a prune left a leftover behind', () => {
        // A catalog member the snippet lane does not want and that no
        // eligibility verdict governs.
        const LEFTOVER = 'test-wf-coord-to-mcp-host'
        const GROK_PROXY = 'test-wf-mcp-host-to-grok-proxy'
        const TWO_TERM_SELECTOR = 'clerum.io/recipe=test-wf,clerum.io/managed-by=wrc'
        type Converged = Awaited<ReturnType<typeof convergedRunLane>>['apiserver']

        function seedLeftover(apiserver: Converged) {
          apiserver.live.set(apiserver.key('sandbox-recipes', LEFTOVER), {
            apiVersion: 'networking.k8s.io/v1',
            kind: 'NetworkPolicy',
            metadata: {
              name: LEFTOVER,
              namespace: 'sandbox-recipes',
              labels: { 'clerum.io/recipe': 'test-wf', 'clerum.io/managed-by': 'wrc' },
            },
            spec: { podSelector: {}, policyTypes: ['Egress'] },
          })
        }

        // nit-3: the WRC suite stubs reconcile() with the real apply and the
        // `reconcile` marker writer; this pins that the real reconcile()
        // publishes exactly that writer's conditions for its own summary.
        it('nit-3: publishes exactly the reconcile marker writer conditions for its own pass summary', async () => {
          const { apiserver, reconciler } = await convergedRunLane()
          seedLeftover(apiserver)
          apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
            code: 403,
            message: 'forbidden',
          })
          const apply = vi.spyOn(
            reconciler as unknown as {
              applyWorkflowNetworkPolicies: (
                ...args: unknown[]
              ) => Promise<NetworkPolicyPassSummary>
            },
            'applyWorkflowNetworkPolicies'
          )

          const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())

          // Liveness witness: the pass ran the run-lane apply once, and its
          // prune could not delete the leftover.
          expect(apply).toHaveBeenCalledTimes(1)
          const summary = await apply.mock.results[0]!.value
          expect(summary).toEqual({
            conflicts: [],
            retryPending: false,
            prune: 'pending',
            legacy: 'removed',
          })
          const published = failed.networkPolicyOwnershipConditions ?? []
          expect(published).toHaveLength(1)
          expect(published).toEqual(
            networkPolicyMarkerConditions(
              { kind: 'reconcile', summary },
              undefined,
              published[0]!.lastTransitionTime!
            )
          )
          expect(failed.networkPolicyRetryPending).toBeUndefined()
        })

        it('keeps it and its PrunePending marker through the retry, and prunes it on the next reconcile()', async () => {
          const { apiserver, reconciler } = await convergedRunLane()
          seedLeftover(apiserver)
          apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
            code: 403,
            message: 'forbidden',
          })

          const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())
          expect(failed.networkPolicyRetryPending).toBeFalsy()
          const published = failed.networkPolicyOwnershipConditions ?? []
          expect(
            published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')
          ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)

          apiserver.api.listNamespacedNetworkPolicy.mockClear()
          apiserver.api.createNamespacedNetworkPolicy.mockClear()
          apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
          expect(
            apiserver.live.delete(apiserver.key('sandbox-recipes', 'test-wf-coord-to-wrc'))
          ).toBe(true)
          const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())
          // Liveness witness: the retry applied the lane the spec wants.
          expect(
            apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
              ([arg]) => arg.body.metadata?.name
            )
          ).toEqual(['test-wf-coord-to-wrc'])
          expect(retried).toEqual({ conflicts: [], retryPending: false })
          expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)

          const pruned = await reconciler.reconcile(
            'test-wf',
            'uid-123',
            'sandbox-recipes',
            spec(),
            {
              conditions: published,
            }
          )
          expect(pruned.networkPolicyRetryPending).toBeFalsy()
          // The prune that landed clears the marker: the pass publishes its
          // conditions and none of them is of that type any more.
          expect(pruned.networkPolicyOwnershipConditions).toBeDefined()
          expect(
            (pruned.networkPolicyOwnershipConditions ?? []).filter(
              c => c.type === 'WorkflowNetworkPoliciesConverged'
            )
          ).toEqual([])
          expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
            namespace: 'sandbox-recipes',
            labelSelector: TWO_TERM_SELECTOR,
          })
          expect(
            apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.map(([arg]) => arg.name)
          ).toEqual([LEFTOVER])
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(false)
        })

        it('R4-L6: publishes PruneUnevaluated, with no retry flag, from a reconcile() whose prune LIST failed', async () => {
          const { apiserver, reconciler } = await convergedRunLane()
          seedLeftover(apiserver)
          const liveList = apiserver.api.listNamespacedNetworkPolicy.getMockImplementation()!
          apiserver.api.listNamespacedNetworkPolicy.mockImplementation(
            async (arg: { namespace: string; labelSelector?: string }) => {
              if (arg.labelSelector === TWO_TERM_SELECTOR) throw { code: 500, message: 'boom' }
              return liveList(arg)
            }
          )
          apiserver.api.listNamespacedNetworkPolicy.mockClear()

          const failed = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', spec())

          // Liveness witness: the prune issued its LIST, and it failed.
          expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledWith({
            namespace: 'sandbox-recipes',
            labelSelector: TWO_TERM_SELECTOR,
          })
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
          expect(failed.networkPolicyRetryPending).toBeFalsy()
          expect(
            (failed.networkPolicyOwnershipConditions ?? []).filter(
              c => c.type === 'WorkflowNetworkPoliciesConverged'
            )
          ).toMatchObject([{ status: 'False', reason: 'PruneUnevaluated' }])

          // Witness: once the LIST answers, a DELETE that does not land is
          // still PrunePending, and it replaces the unevaluated marker.
          apiserver.api.listNamespacedNetworkPolicy.mockImplementation(liveList)
          apiserver.api.deleteNamespacedNetworkPolicy.mockClear()
          apiserver.api.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({
            code: 403,
            message: 'forbidden',
          })
          const pending = await reconciler.reconcile(
            'test-wf',
            'uid-123',
            'sandbox-recipes',
            spec(),
            { conditions: failed.networkPolicyOwnershipConditions ?? [] }
          )
          expect(apiserver.api.deleteNamespacedNetworkPolicy.mock.calls[0]?.[0].name).toBe(LEFTOVER)
          expect(apiserver.live.has(apiserver.key('sandbox-recipes', LEFTOVER))).toBe(true)
          expect(pending.networkPolicyRetryPending).toBeFalsy()
          expect(
            (pending.networkPolicyOwnershipConditions ?? []).filter(
              c => c.type === 'WorkflowNetworkPoliciesConverged'
            )
          ).toMatchObject([{ status: 'False', reason: 'PrunePending' }])
        })

        // The legacy mcp-servers internet policy is deleted by name, apart
        // from the catalog prune: reconcile() deletes it, and the WRC's
        // legacy-retry writer retries it (R4-L4). A failed delete is its own
        // fact (R4-L3), DeletePending on its own condition, with no retry flag
        // and the catalog prune converged, and the next reconcile() pass after
        // the backoff window deletes it again.
        const LEGACY = 'test-wf-mcp-servers-egress-internet'
        const legacyDeletes = (apiserver: Converged) =>
          apiserver.api.deleteNamespacedNetworkPolicy.mock.calls.filter(
            ([arg]) => arg.name === LEGACY
          )

        it('reports a failed legacy internet policy delete from reconcile() as DeletePending, and deletes it on the next reconcile() after its backoff window', async () => {
          const t0 = new Date('2026-09-23T10:00:00.000Z').getTime()
          vi.useFakeTimers({ toFake: ['Date'] })
          vi.setSystemTime(t0)
          try {
            const apiserver = makeApiserverNetworkingApi()
            const reconciler = new WorkflowReconciler(
              makeDeps({ networkingApi: apiserver.api as never })
            )
            const liveDelete = apiserver.api.deleteNamespacedNetworkPolicy.getMockImplementation()!
            apiserver.api.deleteNamespacedNetworkPolicy.mockImplementation(
              async (arg: { name: string; namespace: string }) => {
                if (arg.name === LEGACY && legacyDeletes(apiserver).length === 1) {
                  throw { code: 403, message: 'forbidden' }
                }
                return liveDelete(arg)
              }
            )

            const failed = await reconciler.reconcile(
              'test-wf',
              'uid-123',
              'sandbox-recipes',
              spec()
            )
            expect(legacyDeletes(apiserver)).toHaveLength(1)
            expect(failed.networkPolicyRetryPending).toBeFalsy()
            const published = failed.networkPolicyOwnershipConditions ?? []
            expect(published.filter(c => c.type === 'WorkflowNetworkPoliciesConverged')).toEqual([])
            expect(
              published.filter(c => c.type === 'WorkflowLegacyNetworkPolicyRemoved')
            ).toMatchObject([{ status: 'False', reason: 'DeletePending' }])

            vi.setSystemTime(t0 + 60_000)
            const pruned = await reconciler.reconcile(
              'test-wf',
              'uid-123',
              'sandbox-recipes',
              spec(),
              {
                conditions: published,
              }
            )
            expect(legacyDeletes(apiserver)).toHaveLength(2)
            expect(pruned.networkPolicyRetryPending).toBeFalsy()
            expect(pruned.networkPolicyOwnershipConditions).toEqual([])
          } finally {
            vi.useRealTimers()
          }
        })

        it('sends no legacy internet policy DELETE from the retry, even one no pass has seen gone', async () => {
          const apiserver = makeApiserverNetworkingApi()
          const reconciler = new WorkflowReconciler(
            makeDeps({ networkingApi: apiserver.api as never })
          )

          const retried = await reconciler.retryRunLaneNetworkPolicies('test-wf', 'uid-123', spec())

          // Liveness witness: the retry applied the lanes the spec wants.
          expect(
            apiserver.api.createNamespacedNetworkPolicy.mock.calls
              .map(([arg]) => arg.body.metadata?.name)
              .sort()
          ).toEqual(RUN_LANE_POLICY_NAMES.map(key => key.split('/')[1]).sort())
          expect(retried).toEqual({ conflicts: [], retryPending: false })
          expect(legacyDeletes(apiserver)).toEqual([])
          expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
          expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
        })

        // The retry computes its own verdict, and the verdict decides only
        // which proxy it creates: the retry prunes nothing (see the per-path
        // table on retryRunLaneNetworkPolicies), so a live proxy survives
        // every verdict and one case covers them all. Both cases assert the
        // create of a policy the spec wants as the liveness witness, so a
        // negative assertion cannot pass because the retry never applied.
        describe('with a fresh Codex/Grok verdict', () => {
          const SNIPPET_SIBLING = 'test-wf-snippet-runner-egress'
          const AGENT_LANE = 'test-wf-coord-to-mcp-host'
          const CODEX_PROXY = 'test-wf-mcp-host-to-codex-proxy'
          const projection = (overrides: Record<string, unknown>) => ({
            targets: [],
            eligibleTargets: [],
            derivedScopes: [],
            requiresCodexProxyEgress: false,
            requiresGrokProxyEgress: false,
            driftHashInput: '',
            catalogContentHash: null,
            catalogRevision: null,
            connectionRevision: null,
            eligibility: 'ineligible',
            reason: 'static_only',
            ...overrides,
          })
          const REQUIRED = { eligibility: 'eligible', reason: 'granted' }

          // The proxy policies as the factory emits them for this recipe, stored
          // the way the apiserver returns them.
          function seedProxies(apiserver: Converged) {
            const config = makeConfig()
            const proxies = buildWorkflowNetworkPolicies(
              {
                recipeName: 'test-wf',
                sandboxNamespace: config.sandboxNamespace,
                controlPlaneNamespace: config.controlPlaneNamespace,
                mcpServerNamespace: config.mcpServerNamespace,
                wrcPort: config.wrcPort,
                mcpHostPort: config.mcpHostPort,
                includeMcpHost: true,
                includeCodexProxyEgress: true,
                includeGrokProxyEgress: true,
              },
              []
            ).filter(policy => [CODEX_PROXY, GROK_PROXY].includes(policy.metadata?.name ?? ''))
            expect(proxies.map(policy => policy.metadata?.name).sort()).toEqual([
              CODEX_PROXY,
              GROK_PROXY,
            ])
            for (const policy of proxies) {
              apiserver.live.set(
                apiserver.key('sandbox-recipes', policy.metadata!.name!),
                asApiserverNetworkPolicy({
                  ...policy,
                  metadata: { ...policy.metadata, namespace: 'sandbox-recipes' },
                })
              )
            }
          }

          async function retryWith(
            verdict: { codex: Record<string, unknown>; grok: Record<string, unknown> },
            seed: boolean
          ) {
            const { apiserver, reconciler } = await convergedRunLane()
            apiserver.api.listNamespacedNetworkPolicy.mockClear()
            if (seed) seedProxies(apiserver)
            const verdictSpy = vi.spyOn(reconciler as never, 'codexVerdictFor').mockReturnValue({
              projection: projection(verdict.codex),
              grokProjection: projection(verdict.grok),
            } as never)
            try {
              // An agent recipe with a run id needs mcp-host, the only consumer of
              // the Codex/Grok proxy policies; the snippet spec above never wants them.
              const summary = await reconciler.retryRunLaneNetworkPolicies(
                'test-wf',
                'uid-123',
                makeSpec(),
                'test-wf',
                'run-1'
              )
              expect(verdictSpy).toHaveBeenCalledTimes(1)
              return { apiserver, summary }
            } finally {
              verdictSpy.mockRestore()
            }
          }

          const createdNames = (apiserver: Converged) =>
            apiserver.api.createNamespacedNetworkPolicy.mock.calls.map(
              ([arg]) => arg.body.metadata?.name
            )

          // Decision: the retry never revokes a Codex/Grok proxy. It runs only
          // while a run is in progress or active, and a step of the running run
          // may still be calling the proxy. The retired verdict is the one under
          // which reconcile() deletes the proxy, so it is the case pinned here.
          it('never revokes a live proxy mid-run, whatever the verdict', async () => {
            const { apiserver, summary } = await retryWith({ codex: {}, grok: {} }, true)

            expect(createdNames(apiserver)).toContain(AGENT_LANE)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', SNIPPET_SIBLING))).toBe(true)
            expect(apiserver.api.listNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
            expect(apiserver.api.deleteNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_PROXY))).toBe(true)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(true)
            // A kept proxy is not a pending retry, so the marker can clear.
            expect(summary.retryPending).toBe(false)
          })

          // The verdict-dependent half: a required proxy is created (an allow)
          // and a retired one is not, so a retry that ignores the verdict,
          // creating both or neither, fails here.
          it('creates only the proxy the fresh verdict requires', async () => {
            const { apiserver, summary } = await retryWith(
              { codex: { ...REQUIRED, requiresCodexProxyEgress: true }, grok: {} },
              false
            )

            expect(createdNames(apiserver)).toContain(AGENT_LANE)
            expect(createdNames(apiserver)).toContain(CODEX_PROXY)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', CODEX_PROXY))).toBe(true)
            expect(createdNames(apiserver)).not.toContain(GROK_PROXY)
            expect(apiserver.live.has(apiserver.key('sandbox-recipes', GROK_PROXY))).toBe(false)
            expect(summary.retryPending).toBe(false)
          })
        })
      })
    })
  })

  it('resolves snippet MCP transport workloads into the mounted runner config', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        workloads: [
          {
            id: 'mock-tools',
            type: 'deployment',
            image: 'clerum/mock-mcp-server:test',
            port: 3000,
            transport: { type: 'streamableHttp', path: '/mcp' },
          },
        ],
        steps: [
          {
            id: 'call-mcp',
            run: {
              type: 'snippet',
              language: 'typescript',
              code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 40, b: 2 })',
              capabilities: {
                mcp: {
                  servers: ['mock-tools'],
                  allowedTools: { include: ['mock-tools__add'] },
                },
              },
            },
          },
        ],
      })
    )

    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.mcpServers).toHaveLength(1)
    expect(config.mcpServers[0]).toMatchObject({
      id: 'mock-tools',
      endpoint: expect.stringContaining('.mcp-server.svc.cluster.local:3000/mcp'),
      transport: 'streamableHttp',
    })
    expect(config.steps[0].run.capabilities.mcp.allowedTools.include).toEqual(['mock-tools__add'])
  })

  it('uses the runtime-safe MCP server label in snippet NetworkPolicy selectors for long child recipes', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        config: { ...makeConfig(), enableSnippetRuntime: true } as never,
      })
    )
    const recipeName = 'manual-pr259-layer3a-hybrid-secret-pvc-5step-7f99549a'
    const recipeUid = 'long-child-uid'
    const spec = makeSpec({
      agent: undefined,
      mcpServers: [{ id: 'mock-tools' }],
      workloads: [
        {
          id: 'mock-tools',
          type: 'deployment',
          image: 'clerum/mock-mcp-server:test',
          port: 3000,
          transport: { type: 'streamableHttp', path: '/mcp' },
        },
      ],
      steps: [
        {
          id: 'call-mcp',
          run: {
            type: 'snippet',
            language: 'typescript',
            code: 'return await sdk.mcp.callTool("mock-tools", "add", { a: 40, b: 2 })',
            capabilities: {
              mcp: {
                servers: ['mock-tools'],
                allowedTools: { include: ['mock-tools__add'] },
              },
            },
          },
        },
      ],
    })

    await reconciler.reconcile(recipeName, recipeUid, 'sandbox-recipes', spec)

    const recipeRef = {
      apiVersion: 'clerum.io/v1alpha1' as const,
      kind: 'WorkflowRecipe' as const,
      metadata: { name: recipeName, namespace: 'sandbox-recipes', uid: recipeUid },
      spec,
    }
    const expectedLabel = resolveWorkloadMcpServerLabel(recipeRef, spec.workloads![0])
    const rawInvalidLabel = `${recipeName}-mock-tools`
    const snippetEgress = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === `${recipeName}-snippet-runner-egress`
    )?.[0].body
    const selectedMcpServers = snippetEgress!.spec!.egress!.flatMap(rule =>
      (rule.to ?? [])
        .map(peer => peer.podSelector?.matchLabels?.['clerum.io/mcpserver'])
        .filter(Boolean)
    )

    expect(rawInvalidLabel.length).toBeGreaterThan(63)
    expect(expectedLabel.length).toBeLessThanOrEqual(63)
    expect(selectedMcpServers).toContain(expectedLabel)
    expect(selectedMcpServers).not.toContain(rawInvalidLabel)
  })

  it('resolves hybrid step MCP server IDs and step-level agent settings into coordinator config', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        mcpServers: [
          {
            id: 'mock-tools',
            endpoint: 'http://mock-tools.mcp-server.svc.cluster.local:3000/mcp',
          },
        ],
        steps: [
          { id: 'prepare', run: snippetRun() },
          {
            id: 'calculate',
            dependsOn: ['prepare'],
            instruction: 'Use the add tool.',
            agent: { provider: 'zai', model: 'glm-4.7' },
            mcpServers: ['mock-tools'],
            allowedTools: { include: ['mock-tools__add'] },
            maxIterations: 6,
          },
        ],
      })
    )

    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.agent).toBeUndefined()
    expect(config.steps[1]).toMatchObject({
      id: 'calculate',
      agent: { provider: 'zai', model: 'glm-4.7' },
      mcpServers: [
        {
          name: 'mock-tools',
          url: 'http://mock-tools.mcp-server.svc.cluster.local:3000/mcp',
        },
      ],
      allowedTools: { include: ['mock-tools__add'] },
      maxIterations: 6,
    })
  })

  it('creates coordinator and mcp-host Pods via coreApi', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )
    expect(deps.coreApi.createNamespacedPod).toHaveBeenCalled()
  })

  it('creates a short mcp-host route alias Service for provider callback routing', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    const createdServiceNames = vi
      .mocked(deps.coreApi.createNamespacedService)
      .mock.calls.map(([arg]) => arg.body?.metadata?.name)
    expect(createdServiceNames).toContain(
      buildMcpHostRouteAliasServiceName('test-wf', 'sandbox-recipes')
    )
  })

  describe('headless Services are read before they are written', () => {
    const namespace = 'sandbox-recipes'
    const mcpHostName = buildMcpHostServiceName('test-wf')
    const aliasName = buildMcpHostRouteAliasServiceName('test-wf', namespace)
    const readerName = buildArtifactReaderServiceName('test-wf')
    const snippetRunnerName = buildSnippetRunnerServiceName('test-wf')

    const reconcileTriggeredRun = () =>
      new WorkflowReconciler(deps).reconcile(
        'test-wf',
        'uid-123',
        namespace,
        makeSpec(),
        undefined,
        undefined,
        undefined,
        runId
      )

    it('sends neither POST nor PUT when the live mcp-host and route alias Services match', async () => {
      serveLiveServices(deps, [
        asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
        asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
      ])

      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
        name: mcpHostName,
        namespace,
      })
      expect(deps.coreApi.readNamespacedService).toHaveBeenCalledWith({
        name: aliasName,
        namespace,
      })
      expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
      expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    it('reads, then creates the mcp-host and route alias Services exactly once when they are absent', async () => {
      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
      expect(serviceReadsOf(deps, aliasName)).toBe(1)
      expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
      expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    it('replaces a drifted mcp-host Service with the live resourceVersion and clusterIP', async () => {
      const desired = buildMcpHostHeadlessService('test-wf', namespace)
      // clusterIP differs from the desired 'None' so the assertion below can
      // only pass if the replace carries the live (immutable) value over.
      const drifted = asLiveService(
        {
          ...desired,
          spec: {
            ...desired.spec,
            clusterIP: '10.96.0.42',
            ports: desired.spec?.ports?.slice(0, 1),
          },
        },
        'rv-7'
      )
      serveLiveServices(deps, [
        drifted,
        asLiveService(buildMcpHostRouteAliasHeadlessService('test-wf', namespace), 'rv-2'),
      ])

      await reconcileTriggeredRun()

      expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
      expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
      const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
      expect(replaceCalls).toHaveLength(1)
      const [{ name, namespace: replacedIn, body }] = replaceCalls[0]
      expect(name).toBe(mcpHostName)
      expect(replacedIn).toBe(namespace)
      expect(body.metadata?.resourceVersion).toBe('rv-7')
      expect(body.spec?.clusterIP).toBe('10.96.0.42')
      expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
    })

    describe('when the drift replace conflicts with a concurrent write', () => {
      const driftedMcpHost = (resourceVersion: string) => {
        const desired = buildMcpHostHeadlessService('test-wf', namespace)
        return asLiveService(
          { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
          resourceVersion
        )
      }

      /** Answer successive mcp-host reads in order; a read past the list fails loudly. */
      const serveMcpHostReads = (...responses: Array<LiveService | 404>) => {
        let reads = 0
        vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
          if (name !== mcpHostName) throw { code: 404 }
          const response = responses[reads]
          reads += 1
          if (response === undefined) throw new Error(`unexpected read #${reads} of ${mcpHostName}`)
          if (response === 404) throw { code: 404 }
          return structuredClone(response)
        })
      }

      const conflictOnFirstReplaces = (conflicts: number) => {
        let replaces = 0
        vi.mocked(deps.coreApi.replaceNamespacedService).mockImplementation(async ({ body }) => {
          replaces += 1
          if (replaces <= conflicts) throw { code: 409 }
          return body
        })
      }

      const replaceCalls = () => vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls

      it('re-reads and retries the replace once with the fresh resourceVersion', async () => {
        serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
        conflictOnFirstReplaces(1)

        const result = await reconcileTriggeredRun()

        expect(result.workflowPhase).not.toBe('failed')
        expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
        expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([
          'rv-7',
          'rv-8',
        ])
        expect(replaceCalls()[1][0].body.spec?.ports?.map(port => port.name)).toEqual([
          'http',
          'plugin-sdk',
        ])
      })

      it('stops without a second write when the re-read Service already matches', async () => {
        serveMcpHostReads(
          driftedMcpHost('rv-7'),
          asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-8')
        )
        conflictOnFirstReplaces(1)

        const result = await reconcileTriggeredRun()

        expect(result.workflowPhase).not.toBe('failed')
        expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
        expect(replaceCalls()).toHaveLength(1)
        expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
      })

      it('asks for a fresh pass, without failing, when the Service vanishes after the conflict', async () => {
        serveMcpHostReads(driftedMcpHost('rv-7'), 404)
        conflictOnFirstReplaces(1)

        const result = await reconcileTriggeredRun()

        expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
        expect(replaceCalls()).toHaveLength(1)
        expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
        expect(result.workflowPhase).not.toBe('failed')
        expect(result.phase).not.toBe('failed')
        expect(result.skipStatusPatch).toBe(true)
        expect(result.message).toContain(
          `Headless Service "${mcpHostName}" disappeared after a replace conflict`
        )
      })

      it('asks for a fresh pass, without failing, when the replace conflicts on both attempts', async () => {
        serveMcpHostReads(driftedMcpHost('rv-7'), driftedMcpHost('rv-8'))
        conflictOnFirstReplaces(2)

        const result = await reconcileTriggeredRun()

        expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
        expect(replaceCalls().map(([arg]) => arg.body.metadata?.resourceVersion)).toEqual([
          'rv-7',
          'rv-8',
        ])
        expect(result.workflowPhase).not.toBe('failed')
        expect(result.phase).not.toBe('failed')
        expect(result.skipStatusPatch).toBe(true)
        expect(result.message).toContain(
          `Headless Service "${mcpHostName}" replace conflicted on both attempts`
        )
      })

      it('propagates a replace error other than a conflict without retrying', async () => {
        serveMcpHostReads(driftedMcpHost('rv-7'))
        vi.mocked(deps.coreApi.replaceNamespacedService).mockRejectedValue(
          Object.assign(new Error('HTTP-Code: 422 Message: Service is invalid'), { code: 422 })
        )

        const result = await reconcileTriggeredRun()

        expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
        expect(replaceCalls()).toHaveLength(1)
        expect(result.workflowPhase).toBe('failed')
        expect(result.message).toContain('HTTP-Code: 422')
      })
    })

    it('leaves a drifted route alias Service untouched because it is create-only', async () => {
      const desiredAlias = buildMcpHostRouteAliasHeadlessService('test-wf', namespace)
      serveLiveServices(deps, [
        asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-1'),
        asLiveService(
          {
            ...desiredAlias,
            spec: { ...desiredAlias.spec, ports: desiredAlias.spec?.ports?.slice(0, 1) },
          },
          'rv-2'
        ),
      ])

      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(serviceReadsOf(deps, aliasName)).toBe(1)
      expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    it('re-reads and converges when another writer creates the mcp-host Service after the read', async () => {
      const live = asLiveService(buildMcpHostHeadlessService('test-wf', namespace), 'rv-9')
      let mcpHostReads = 0
      vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
        if (name !== mcpHostName) throw { code: 404 }
        mcpHostReads += 1
        if (mcpHostReads === 1) throw { code: 404 }
        return structuredClone(live)
      })
      vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
        if (body.metadata?.name === mcpHostName) throw { code: 409 }
        return body
      })

      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
      expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    it('repairs a drifted mcp-host Service that another writer created after the read', async () => {
      const desired = buildMcpHostHeadlessService('test-wf', namespace)
      const racedDrifted = asLiveService(
        { ...desired, spec: { ...desired.spec, ports: desired.spec?.ports?.slice(0, 1) } },
        'rv-11'
      )
      let mcpHostReads = 0
      vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
        if (name !== mcpHostName) throw { code: 404 }
        mcpHostReads += 1
        if (mcpHostReads === 1) throw { code: 404 }
        return structuredClone(racedDrifted)
      })
      vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
        if (body.metadata?.name === mcpHostName) throw { code: 409 }
        return body
      })

      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
      const replaceCalls = vi.mocked(deps.coreApi.replaceNamespacedService).mock.calls
      expect(replaceCalls).toHaveLength(1)
      const [{ name, body }] = replaceCalls[0]
      expect(name).toBe(mcpHostName)
      expect(body.metadata?.resourceVersion).toBe('rv-11')
      expect(body.spec?.ports?.map(port => port.name)).toEqual(['http', 'plugin-sdk'])
    })

    it('asks for a fresh pass, without failing, when the mcp-host Service vanishes after a create conflict', async () => {
      vi.mocked(deps.coreApi.readNamespacedService).mockRejectedValue({ code: 404 })
      vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
        if (body.metadata?.name === mcpHostName) throw { code: 409 }
        return body
      })

      const result = await reconcileTriggeredRun()

      expect(serviceReadsOf(deps, mcpHostName)).toBe(2)
      expect(createdServiceNamesOf(deps).filter(name => name === mcpHostName)).toHaveLength(1)
      expect(result.workflowPhase).not.toBe('failed')
      expect(result.phase).not.toBe('failed')
      expect(result.skipStatusPatch).toBe(true)
      expect(result.message).toContain(
        `Headless Service "${mcpHostName}" disappeared after create conflict`
      )
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    it('propagates a non-404 read error instead of creating the mcp-host Service blind', async () => {
      const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
        code: 403,
      })
      vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
        if (name === mcpHostName) throw forbidden
        throw { code: 404 }
      })

      const result = await reconcileTriggeredRun()

      expect(serviceReadsOf(deps, mcpHostName)).toBe(1)
      expect(result.workflowPhase).toBe('failed')
      expect(result.message).toContain('HTTP-Code: 403')
      expect(createdServiceNamesOf(deps)).not.toContain(mcpHostName)
    })

    it('propagates a non-404 read error instead of creating a create-only Service blind', async () => {
      const forbidden = Object.assign(new Error('HTTP-Code: 403 Message: services is forbidden'), {
        code: 403,
      })
      vi.mocked(deps.coreApi.readNamespacedService).mockImplementation(async ({ name }) => {
        if (name === aliasName) throw forbidden
        throw { code: 404 }
      })

      const result = await reconcileTriggeredRun()

      // The mcp-host Service is handled first and was created, so the pass reached the alias.
      expect(createdServiceNamesOf(deps)).toContain(mcpHostName)
      expect(serviceReadsOf(deps, aliasName)).toBe(1)
      expect(result.workflowPhase).toBe('failed')
      expect(result.message).toContain('HTTP-Code: 403')
      expect(createdServiceNamesOf(deps)).not.toContain(aliasName)
    })

    it('converges when another writer creates a create-only Service between the read and the POST', async () => {
      vi.mocked(deps.coreApi.createNamespacedService).mockImplementation(async ({ body }) => {
        if (body.metadata?.name === aliasName) throw { code: 409 }
        return body
      })

      const result = await reconcileTriggeredRun()

      expect(result.workflowPhase).not.toBe('failed')
      expect(serviceReadsOf(deps, aliasName)).toBe(1)
      expect(createdServiceNamesOf(deps).filter(name => name === aliasName)).toHaveLength(1)
      expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
    })

    describe('snippet-runner and artifact-reader Services', () => {
      const reconcileSnippetWorkflow = () => {
        vi.mocked(deps.coreApi.readNamespacedSecret).mockResolvedValue({
          metadata: { labels: { 'clerum.io/shared': 'true' } },
          data: { key: 'dmFsdWU=' },
        })
        return new WorkflowReconciler(deps).reconcile(
          'test-wf',
          'uid-123',
          namespace,
          makeSpec({
            agent: undefined,
            workloads: [{ id: 'postgres', type: 'deployment', image: 'postgres:16', port: 5432 }],
            steps: [
              {
                id: 'snippet',
                run: {
                  type: 'snippet',
                  language: 'typescript',
                  code: 'return await sdk.postgres.query({ workload: "postgres", database: "clerum" }, { sql: "select 1" })',
                  capabilities: {
                    secrets: [
                      { alias: 'api_key', secretRef: { name: 'public-api-key', key: 'key' } },
                    ],
                    postgres: { access: 'read', workloads: ['postgres'] },
                  },
                },
              },
            ],
          })
        )
      }

      beforeEach(() => {
        deps = makeDeps({
          config: {
            ...makeConfig(),
            enableSnippetRuntime: true,
            snippetRunnerImage: 'clerum/workflow-snippet-runner:test',
          } as never,
        })
      })

      it('sends no POST for snippet-runner and artifact-reader Services that already exist', async () => {
        serveLiveServices(deps, [
          asLiveService(buildArtifactReaderHeadlessService('test-wf', namespace), 'rv-3'),
          asLiveService(buildSnippetRunnerHeadlessService('test-wf', namespace), 'rv-4'),
        ])

        const result = await reconcileSnippetWorkflow()

        expect(result.workflowPhase).not.toBe('failed')
        expect(serviceReadsOf(deps, readerName)).toBe(1)
        expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
        expect(createdServiceNamesOf(deps)).not.toContain(readerName)
        expect(createdServiceNamesOf(deps)).not.toContain(snippetRunnerName)
        expect(deps.coreApi.replaceNamespacedService).toHaveBeenCalledTimes(0)
      })

      it('reads, then creates snippet-runner and artifact-reader Services when they are absent', async () => {
        const result = await reconcileSnippetWorkflow()

        expect(result.workflowPhase).not.toBe('failed')
        expect(serviceReadsOf(deps, readerName)).toBe(1)
        expect(serviceReadsOf(deps, snippetRunnerName)).toBe(1)
        expect(createdServiceNamesOf(deps).filter(name => name === readerName)).toHaveLength(1)
        expect(createdServiceNamesOf(deps).filter(name => name === snippetRunnerName)).toHaveLength(
          1
        )
      })
    })
  })

  it('applies runtime NetworkPolicies before creating runtime Pods', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    const firstPolicyCall = vi.mocked(deps.networkingApi.createNamespacedNetworkPolicy).mock
      .invocationCallOrder[0]
    const runtimePodCall = vi
      .mocked(deps.coreApi.createNamespacedPod)
      .mock.calls.findIndex(
        ([arg]) => arg.body?.metadata?.name !== 'test-wf-workflow-output-anchor'
      )
    expect(runtimePodCall).toBeGreaterThanOrEqual(0)
    const firstRuntimePodCall = vi.mocked(deps.coreApi.createNamespacedPod).mock
      .invocationCallOrder[runtimePodCall]
    expect(firstPolicyCall).toBeLessThan(firstRuntimePodCall)
  })

  it('registers an agentic parent workflow without runtime pods until a run id exists', async () => {
    const coreApi = makeCoreApi(false, { workflowOutputAnchorPhase: null }) as ReturnType<
      typeof makeCoreApi
    >
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({ coreApi: coreApi as never, networkingApi: networkingApi as never })
    )

    const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())

    expect(result).toMatchObject({
      phase: 'active',
      message: 'Workflow trigger infrastructure registered (workflow-agentic)',
      clearWorkflowExecution: true,
    })
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toEqual(['test-wf-workflow-output-anchor'])
    expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
  })

  it('threads workflow run id into the coordinator pod env', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    const pod = getCreatedCoordinatorPod(coreApi)
    const env = pod.spec!.containers![0].env!.find(
      (e: { name: string }) => e.name === 'CLERUM_WORKFLOW_RUN_ID'
    )
    expect(env!.value).toBe(runId)
  })

  it('maps admin workflow actors to usage-only admin-ui user keys', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId,
      'control-plane-admin-ui',
      '11111111-1111-4111-8111-111111111111',
      'admin'
    )

    const pod = getCreatedCoordinatorPod(coreApi)
    const env = pod.spec!.containers![0].env!
    expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_RUN_ID')!.value).toBe(
      runId
    )
    expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_TEAM_ID')!.value).toBe(
      'control-plane-admin-ui'
    )
    expect(env.find((e: { name: string }) => e.name === 'CLERUM_WORKFLOW_USER_ID')!.value).toBe(
      'admin-ui/11111111-1111-4111-8111-111111111111'
    )
  })

  it('accepts pure snippet workflows and skips the mcp-host pod', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    expect(result.phase).toBe('deploying')
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-coordinator')
    expect(createdPodNames).not.toContain('test-wf-mcp-host')
    expect(result.message).not.toContain('not yet implemented')
  })

  it('rejects custom coordinator images when the feature flag is disabled', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('custom coordinator images are disabled')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('runs pure custom coordinator workflows without mcp-host and with implicit platform output', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
      })
    )

    expect(result.message).toContain('workflow-custom')
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-coordinator')
    expect(createdPodNames).toContain('test-wf-artifact-reader')
    expect(createdPodNames).not.toContain('test-wf-mcp-host')

    const coordPod = getCreatedCoordinatorPod(coreApi)
    const artifactReaderPod = getCreatedArtifactReaderPod(coreApi)
    const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
    expect(coordPod.spec!.containers![0].image).toBe('clerum/workflow-custom-sdk-e2e:test')
    expect(artifactReaderPod.spec!.containers![0].image).toBe('clerum/workflow-recipes:test')
    const artifactReaderOutputMount = artifactReaderPod.spec!.containers![0].volumeMounts!.find(
      (mount: { mountPath: string }) => mount.mountPath === '/output'
    )
    expect(artifactReaderOutputMount).toMatchObject({
      name: 'recipe-output',
      mountPath: '/output',
      subPath: 'workflow-output/test-wf',
    })
    expect(artifactReaderOutputMount).not.toHaveProperty('readOnly')
    expect(
      artifactReaderPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expect(envNames).not.toContain('MCP_HOST_TOKEN')
    expect(envNames).not.toContain('MCP_HOST_RUNTIME_ACCESS_TOKEN')
    expect(coordPod.spec!.enableServiceLinks).toBe(false)
    expect(coordPod.spec!.hostNetwork).toBe(false)
    expect(coordPod.spec!.hostPID).toBe(false)
    expect(coordPod.spec!.hostIPC).toBe(false)
    expect(coordPod.spec!.activeDeadlineSeconds).toBe(3300)
    expect(
      coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'tmp')!.emptyDir
    ).toEqual({
      sizeLimit: '64Mi',
    })
    expect(
      coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expect(
      coordPod.spec!.containers![0].volumeMounts!.find(
        (m: { mountPath: string }) => m.mountPath === '/output'
      )!.subPath
    ).toBe('workflow-output/test-wf')
    expectWorkflowOutputAnchorAffinity(coordPod)
    expectWorkflowOutputAnchorAffinity(artifactReaderPod)
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
    expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()

    const coordinatorSecret = coreApi.createNamespacedSecret.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'wf-test-wf-coordinator-token'
    )?.[0].body
    expect(coordinatorSecret!.data!['mcp-host-token']).toBeUndefined()
    expect(Buffer.from(coordinatorSecret!.data!['wrc-token'], 'base64').toString()).toBe(
      'custom-coord-wrc-token'
    )
  })

  it('honors explicit custom coordinator pvc output while keeping the mount run scoped', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
        output: { destination: 'pvc' },
      }),
      undefined,
      undefined,
      undefined,
      runId
    )

    const coordPod = getCreatedCoordinatorPod(coreApi)
    const outputMount = coordPod.spec!.containers![0].volumeMounts!.find(
      (m: { mountPath: string }) => m.mountPath === '/output'
    )
    expect(outputMount).toMatchObject({
      name: 'recipe-output',
      mountPath: '/output',
      subPath: `workflow-output/test-wf/${runId}`,
    })
    expect(
      coordPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
  })

  it('propagates runtimeEgress and grants declared public HTTP egress to custom coordinators', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const resolveRuntimeHttpEgressCidrs = vi.fn().mockResolvedValue(['93.184.216.34/32'])
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        runtimeEgress: { http: { allowedHosts: ['example.com'] } },
        steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
        output: { destination: 'pvc' },
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(resolveRuntimeHttpEgressCidrs).toHaveBeenCalledWith(['example.com'])
    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.runtimeEgress.http.allowedHosts).toEqual(['example.com'])

    const coordPolicy = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-coord-to-wrc'
    )?.[0].body
    expect(coordPolicy).toBeDefined()
    expect(coordPolicy!.metadata!.labels!['clerum.io/egress-class']).toBe('exact-host')
    expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(true)
    expect(publicHttpEgressCidrs(coordPolicy!)).toEqual(['93.184.216.34/32'])
  })

  it('grants explicit public-web egress to custom coordinators without DNS resolution', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const resolveRuntimeHttpEgressCidrs = vi.fn()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        resolveRuntimeHttpEgressCidrs,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        runtimeEgress: { http: { egressClass: 'public-web' } },
        steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
        output: { destination: 'pvc' },
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(resolveRuntimeHttpEgressCidrs).not.toHaveBeenCalled()
    const coordPolicy = networkingApi.createNamespacedNetworkPolicy.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-coord-to-wrc'
    )?.[0].body
    expect(coordPolicy).toBeDefined()
    expect(coordPolicy!.metadata!.labels!['clerum.io/egress-class']).toBe('public-web')
    expect(hasPublicHttpEgressRule(coordPolicy!)).toBe(true)
    expect(publicHttpEgressCidrs(coordPolicy!)).toEqual(['0.0.0.0/0'])
  })

  it('keeps artifact-reader available for broker-backed custom coordinator output artifacts', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        output: { destination: 'pvc' },
        steps: [
          { id: 'prepare' },
          {
            id: 'broker-review',
            dependsOn: ['prepare'],
            instruction: 'Use the broker',
            agent: { provider: 'zai', model: 'glm-4.7' },
            mcpServers: ['mock-tools'],
          },
          { id: 'emit', dependsOn: ['broker-review'] },
        ],
      }),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.message).toContain('workflow-custom')
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-coordinator')
    expect(createdPodNames).toContain('test-wf-mcp-host')
    expect(createdPodNames).toContain('test-wf-artifact-reader')
    expect(getCreatedArtifactReaderPod(coreApi).spec!.containers![0].volumeMounts).toContainEqual(
      expect.objectContaining({
        name: 'recipe-output',
        mountPath: '/output',
      })
    )
    expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCoordinatorToMcpHostToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
  })

  it('persists resolved inputs into the mounted custom coordinator config', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const resolvedInputs = {
      requestId: 'ui-custom-123',
      approvalThreshold: 1000,
      scenario: 'desktop-custom-coordinator-trigger',
    }

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        inputContract: {
          properties: {
            requestId: { type: 'string', default: 'ui-custom-default' },
            approvalThreshold: { type: 'number', default: 5000 },
            scenario: { type: 'string', default: 'default-scenario' },
          },
        },
        steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
        output: { destination: 'pvc' },
      }),
      undefined,
      resolvedInputs
    )

    const configMap = coreApi.createNamespacedConfigMap.mock.calls.find(
      ([arg]) => arg.body?.metadata?.name === 'test-wf-workflow-config'
    )?.[0].body
    const config = JSON.parse(configMap!.data!['config.json'])
    expect(config.inputs).toEqual(resolvedInputs)
    expect(config.inputContract.properties.requestId.default).toBe('ui-custom-default')
  })

  it('rejects custom coordinator timeout budgets above the active deadline cap', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [
          { id: 'prepare', timeoutSeconds: 2400 },
          { id: 'emit', dependsOn: ['prepare'], timeoutSeconds: 2400 },
        ],
        output: { destination: 'pvc' },
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('maximum active deadline')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('runs broker-backed custom coordinator workflows with mcp-host credentials only when needed', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: { provider: 'openai', model: 'gpt-4o-mini' },
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [
          { id: 'prepare' },
          { id: 'review', dependsOn: ['prepare'], instruction: 'Review the prepared result.' },
        ],
        output: { destination: 'pvc' },
      }),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.message).toContain('workflow-custom')
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-coordinator')
    expect(createdPodNames).toContain('test-wf-mcp-host')
    const coordPod = getCreatedCoordinatorPod(coreApi)
    const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
    expect(envNames).toContain('CLERUM_MCPHOST_URL')
    expect(envNames).toContain('MCP_HOST_TOKEN_FILE')
    expect(envNames).not.toContain('MCP_HOST_TOKEN')
    for (const forbiddenEnv of [
      'MCP_HOST_RUNTIME_ACCESS_TOKEN',
      'MCP_HOST_RUNTIME_REFRESH_TOKEN',
      'MCP_HOST_WORKFLOW_CONTROL_TOKEN',
      'MCP_HOST_GATEWAY_URL',
      'OPENAI_API_KEY',
      'ZAI_API_KEY',
      'BAILIAN_API_KEY',
      'CLAUDE_API_KEY',
      'CLERUM_MODEL_API_KEY',
    ]) {
      expect(envNames).not.toContain(forbiddenEnv)
    }
    expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCoordinatorToMcpHostToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
  })

  it('keeps broker-backed custom workflows without explicit pvc output on WRC-managed output', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: { provider: 'openai', model: 'gpt-4o-mini' },
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [
          { id: 'prepare' },
          { id: 'review', dependsOn: ['prepare'], instruction: 'Review the prepared result.' },
        ],
      }),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.message).toContain('workflow-custom')
    const mcpHostPod = getCreatedMcpHostPod(coreApi)
    const outputMount = mcpHostPod.spec!.containers![0].volumeMounts!.find(
      (m: { mountPath: string }) => m.mountPath === '/output'
    )
    expect(outputMount!.subPath).toBe(`workflow-output/test-wf/${runId}`)
    expect(
      mcpHostPod.spec!.volumes!.find((v: { name: string }) => v.name === 'recipe-output')!
        .persistentVolumeClaim!.claimName
    ).toBe('test-wf-workflow-output')
    expectWorkflowOutputAnchorAffinity(mcpHostPod)
    expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalled()
  })

  it('keeps existing custom coordinator token Secrets reduced on later reconciles', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
      return {}
    })
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
              'base64'
            ),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }, { id: 'emit', dependsOn: ['prepare'] }],
        output: { destination: 'pvc' },
      })
    )

    expect(result.message).toContain('workflow-custom')
    expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
    expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledTimes(1)
    expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
  })

  it('prunes stale mcp-host tokens from existing pure custom coordinator Secrets', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
      return {}
    })
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'mcp-host-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
            'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
              'base64'
            ),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
        output: { destination: 'pvc' },
      })
    )

    expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        body: {
          data: {
            'mcp-host-token': null,
          },
        },
      }),
      expect.any(Object)
    )
  })

  it('rotates an existing coordinator WRC token when a custom coordinator needs the custom subject', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
      return {}
    })
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'coordinator' })).toString(
              'base64'
            ),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
        output: { destination: 'pvc' },
      })
    )

    expect(tokenFactory.signCustomCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCoordinatorToWrcToken).not.toHaveBeenCalled()
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        body: {
          data: {
            'wrc-token': Buffer.from('custom-coord-wrc-token').toString('base64'),
          },
        },
      }),
      expect.any(Object)
    )
  })

  it('rotates an existing custom coordinator WRC token back to the builtin coordinator subject', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-coordinator-token') throw { code: 409 }
      return {}
    })
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'mcp-host-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
            'wrc-token': Buffer.from(makeJwtWithExp(3600, { sub: 'custom-coordinator' })).toString(
              'base64'
            ),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
    )

    await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())

    expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCustomCoordinatorToWrcToken).not.toHaveBeenCalled()
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        body: {
          data: {
            'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
          },
        },
      }),
      expect.any(Object)
    )
  })

  it('refreshes coordinator runtime credentials during steady-state snippet workflows', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'wrc-token': Buffer.from(
              makeJwtWithExp(30, { sub: 'coordinator', recipeName: 'test-wf' })
            ).toString('base64'),
            'snippet-runner-token': Buffer.from('snippet-runner-token').toString('base64'),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
    )

    await reconciler.ensureCoordinatorRuntimeCredentials(
      'sandbox-recipes',
      'test-wf',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
      })
    )

    expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(tokenFactory.signCoordinatorToMcpHostToken).not.toHaveBeenCalled()
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        namespace: 'sandbox-recipes',
        body: {
          data: {
            'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
          },
        },
      }),
      expect.any(Object)
    )
  })

  it('refreshes coordinator GFS access when declared publish scopes change', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'gfs-token': makeEncodedGfsAccess(3600, 'test-wf', ['gfs.read']),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureCoordinatorRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({
        gfs: {
          publishTargets: [{ drive: 'main', target: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' }],
        },
      })
    )

    expect(mintRecipeHostGfsToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', {
      scopes: ['gfs.read', 'gfs.write'],
    })
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        namespace: 'recipe-runtime-ns',
        body: {
          data: expect.objectContaining({
            'gfs-token': expect.any(String),
          }),
        },
      }),
      expect.any(Object)
    )
  })

  it('refreshes short-TTL coordinator tokens early enough for Secret volume projection', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const tokenFactory = makeTokenFactory()
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-coordinator-token') {
        return {
          data: {
            'wrc-token': Buffer.from(
              makeJwtWithExp(60, { sub: 'coordinator', recipeName: 'test-wf' })
            ).toString('base64'),
            'snippet-runner-token': Buffer.from('snippet-runner-token').toString('base64'),
          },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        tokenFactory: tokenFactory as never,
        config: {
          ...makeConfig(),
          runtimeTokenTtlSeconds: 90,
          runtimeTokenRefreshBeforeSeconds: 45,
        } as never,
      })
    )

    await reconciler.ensureCoordinatorRuntimeCredentials(
      'sandbox-recipes',
      'test-wf',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'snippet', run: snippetRun() }],
      })
    )

    expect(tokenFactory.signCoordinatorToWrcToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-coordinator-token',
        namespace: 'sandbox-recipes',
        body: {
          data: {
            'wrc-token': Buffer.from('coord-wrc-token').toString('base64'),
          },
        },
      }),
      expect.any(Object)
    )
  })

  it('rejects coordinator image changes after the runtime pod already exists', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'test-wf-coordinator') {
        return {
          status: { phase: 'Running' },
          spec: { containers: [{ image: 'clerum/coordinator:test' }] },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('coordinator image is immutable')
    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('allows built-in coordinator pods to finish after platform image config changes', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'test-wf-coordinator') {
        return {
          metadata: { labels: { 'clerum.io/coordinator-tier': 'builtin' } },
          status: { phase: 'Running' },
          spec: { containers: [{ image: 'clerum/coordinator:old' }] },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          coordinatorImage: 'clerum/coordinator:new',
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(result.message).not.toContain('coordinator image is immutable')
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).not.toContain('test-wf-coordinator')
  })

  it('rejects removing a custom coordinator image after the custom runtime pod already exists', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'test-wf-coordinator') {
        return {
          metadata: { labels: { 'clerum.io/coordinator-tier': 'custom' } },
          status: { phase: 'Running' },
          spec: { containers: [{ image: 'clerum/workflow-custom-sdk-e2e:test' }] },
        }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('coordinator tier is immutable')
    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects disallowed custom coordinator image prefixes before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'docker.io/unknown/custom:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('not allowed')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects adjacent custom coordinator image prefix bypasses', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/workflow'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'ghcr.io/your-org/workflow-evil:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('not allowed')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects custom coordinator image path traversal segments before prefix checks', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['ghcr.io/your-org/'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'ghcr.io/your-org/../../attacker/workflow:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('custom coordinator image reference is invalid')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects custom coordinator images when the allowlist is empty', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: [],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('allowlist is empty')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it.each([
    {
      caseName: 'slash boundary',
      prefixes: ['ghcr.io/your-org'],
      image: 'ghcr.io/your-org/workflow-custom:test',
    },
    {
      caseName: 'tag boundary',
      prefixes: ['clerum/workflow-custom-sdk-e2e'],
      image: 'clerum/workflow-custom-sdk-e2e:test',
    },
    {
      caseName: 'digest boundary',
      prefixes: ['clerum/workflow-custom-sdk-e2e'],
      image: `clerum/workflow-custom-sdk-e2e@sha256:${'a'.repeat(64)}`,
    },
    {
      caseName: 'prefix ending with explicit tag delimiter',
      prefixes: ['clerum/workflow-custom-sdk-e2e:'],
      image: 'clerum/workflow-custom-sdk-e2e:test',
    },
  ])('accepts custom coordinator allowlist $caseName', async ({ prefixes, image }) => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: prefixes,
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: image,
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).not.toBe('failed')
    expect(coreApi.createNamespacedPod).toHaveBeenCalled()
  })

  it('rejects latest custom coordinator images', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:latest',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('must not use :latest')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects latest custom coordinator images even when a digest is present', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: `clerum/workflow-custom-sdk-e2e:latest@sha256:${'a'.repeat(64)}`,
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('must not use :latest')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects malformed custom coordinator digests', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test@sha256:nothex',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('valid sha256 digest')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects uppercase custom coordinator sha256 digests', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/'],
          requireCoordinatorImageDigest: true,
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: `clerum/workflow-custom-sdk-e2e:test@sha256:${'A'.repeat(64)}`,
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('valid sha256 digest')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('enforces custom coordinator digest policy when configured', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/'],
          requireCoordinatorImageDigest: true,
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('valid sha256 digest')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('accepts valid custom coordinator digest refs when digest policy is configured', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e'],
          requireCoordinatorImageDigest: true,
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: `clerum/workflow-custom-sdk-e2e:test@sha256:${'a'.repeat(64)}`,
        steps: [{ id: 'prepare' }],
      })
    )

    expect(result.message).toContain('workflow-custom')
    expect(coreApi.createNamespacedPod).toHaveBeenCalled()
  })

  it('fails custom approval steps without a broker agent before creating runtime resources', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          enableCustomCoordinatorImage: true,
          allowedCoordinatorImagePrefixes: ['clerum/workflow-custom-sdk-e2e:'],
        } as never,
      })
    )

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        coordinatorImage: 'clerum/workflow-custom-sdk-e2e:test',
        steps: [
          {
            id: 'approval',
            requiresApproval: {
              target: { userId: 'operator' },
              message: 'approve custom step',
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('requires an agent configuration for approval')
    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('classifies all-run workflows as snippet workflows even when a default agent is present', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(result.message).toContain('workflow-snippet')
    expect(createdPodNames).toContain('test-wf-coordinator')
    expect(createdPodNames).not.toContain('test-wf-mcp-host')
  })

  it('rejects run steps that also configure a step agent', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'ambiguous',
            run: snippetRun(),
            agent: { provider: 'zai', model: 'glm-4.7' },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('cannot configure an agent when run is set')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects snippet run steps that require approval before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'gated',
            run: snippetRun(),
            requiresApproval: {
              target: { userId: 'operator' },
              message: 'approve snippet step',
            },
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('cannot require approval')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('skips runtime token Secret provisioning for pure snippet workflows', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const reconciler = new WorkflowReconciler(deps)

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
  })

  it('does not mount mcp-host credentials on snippet coordinator pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'render', run: snippetRun() }],
      })
    )

    const coordPod = getCreatedCoordinatorPod(coreApi)
    const envNames = coordPod.spec!.containers![0].env!.map((env: { name: string }) => env.name)
    expect(envNames).not.toContain('MCP_HOST_ENDPOINT')
    expect(envNames).not.toContain('MCP_HOST_TOKEN')
    expect(envNames).not.toContain('MCP_HOST_RUNTIME_ACCESS_TOKEN')
  })

  it('mounts the parent output PVC on snippet coordinator pods when workflow output is enabled', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'pdf',
            run: snippetRun(),
          },
        ],
      }),
      undefined,
      undefined,
      undefined,
      runId
    )

    const coordPod = getCreatedCoordinatorPod(coreApi)
    const outputMount = coordPod.spec!.containers![0].volumeMounts!.find(
      (mount: { mountPath: string }) => mount.mountPath === '/output'
    )
    expect(outputMount).toMatchObject({
      name: 'recipe-output',
      subPath: `workflow-output/test-wf/${runId}`,
    })
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-artifact-reader')
    expect(createdPodNames).not.toContain('test-wf-mcp-host')
    expect(getCreatedArtifactReaderPod(coreApi).spec!.containers![0].volumeMounts).toContainEqual(
      expect.objectContaining({
        name: 'recipe-output',
        mountPath: '/output',
        subPath: `workflow-output/test-wf/${runId}`,
      })
    )
    expectWorkflowOutputAnchorAffinity(coordPod)
  })

  it('rejects invalid snippet step shapes before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'bad', instruction: 'run', run: snippetRun() }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('cannot configure both run and instruction')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects unsupported snippet run fields before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'bad',
            run: { ...snippetRun(), unexpected: true } as never,
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('run contains unsupported field "unexpected"')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects non-snippet run types before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          {
            id: 'bad',
            run: { type: 'handler', handler: 'foo' } as never,
          },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('run.type must be snippet')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects cyclic step dependencies before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          { id: 'first', dependsOn: ['second'], run: snippetRun() },
          { id: 'second', dependsOn: ['first'], run: snippetRun() },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('dependency cycle detected')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects duplicate step ids before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [
          { id: 'first', run: { handler: 'noop' } },
          { id: 'first', run: { handler: 'noop' } },
        ],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('duplicate step id "first" is not allowed')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('rejects unknown step dependencies before creating pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        agent: undefined,
        steps: [{ id: 'first', dependsOn: ['missing'], run: snippetRun() }],
      })
    )

    expect(result.workflowPhase).toBe('failed')
    expect(result.message).toContain('depends on unknown step "missing"')
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('does not read selected-node or pin mcp-host pods to stale node names', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    const mcpHostPod = getCreatedMcpHostPod(coreApi)
    expectWorkflowOutputAnchorAffinity(mcpHostPod)
  })

  it('creates mcp-host pods with anchor affinity and does not inspect selected-node', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: { annotations: {} },
      spec: { accessModes: ['ReadWriteOnce'] },
      status: { accessModes: ['ReadWriteOnce'] },
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).toBeDefined()
    expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    const mcpHostPod = getCreatedMcpHostPod(coreApi)
    expectWorkflowOutputAnchorAffinity(mcpHostPod)
  })

  it('does not need selected-node lookup to use anchor affinity', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockRejectedValue({ code: 404 })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).toBeDefined()
    expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    const mcpHostPod = getCreatedMcpHostPod(coreApi)
    expectWorkflowOutputAnchorAffinity(mcpHostPod)
  })

  it('ignores stale selected-node annotations when creating mcp-host pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPersistentVolumeClaim = vi.fn().mockResolvedValue({
      metadata: {
        annotations: {
          'volume.kubernetes.io/selected-node': 'node-a',
        },
      },
      spec: { accessModes: ['ReadWriteMany'] },
      status: { accessModes: ['ReadWriteMany'] },
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).toBeDefined()
    expect(coreApi.readNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    const mcpHostPod = getCreatedMcpHostPod(coreApi)
    expectWorkflowOutputAnchorAffinity(mcpHostPod)
  })

  it('creates NetworkPolicies via networkingApi', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(deps.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalled()
  })

  it('passes runRetention through schedule reconcile into workflow_schedules', async () => {
    const { pool: pgPool, query: pgQuery } = makePgPool()
    const schedulingDeps = makeDeps({ pgPool })
    const reconciler = new WorkflowReconciler(schedulingDeps)

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        triggers: {
          schedule: {
            cron: '0 9 * * *',
            timezone: 'UTC',
          },
        },
        runRetention: {
          maxRunDurationSeconds: 1800,
          ttlSecondsAfterFinished: 3600,
        },
      }),
      undefined,
      undefined,
      undefined,
      undefined,
      '11111111-1111-4111-8111-111111111111'
    )

    const insertCall = pgQuery.mock.calls.find(([sql]) =>
      String(sql).includes('INSERT INTO workflow_schedules')
    )
    expect(insertCall).toBeDefined()
    expect(String(insertCall?.[0])).toContain('max_duration_seconds')
    expect(String(insertCall?.[0])).toContain('ttl_seconds_after_finished')
    expect((insertCall?.[1] as unknown[] | undefined)?.[2]).toBe(
      '11111111-1111-4111-8111-111111111111'
    )
    expect((insertCall?.[1] as unknown[] | undefined)?.[8]).toBe(1800)
    expect((insertCall?.[1] as unknown[] | undefined)?.[9]).toBe(3600)
  })

  it('mirrors triggers.schedule.concurrencyPolicy into workflow_schedules', async () => {
    const { pool: pgPool, query: pgQuery } = makePgPool()
    const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({
        triggers: {
          schedule: { cron: '*/5 * * * *', timezone: 'UTC', concurrencyPolicy: 'Allow' },
        },
      }),
      undefined,
      undefined,
      undefined,
      undefined,
      '11111111-1111-4111-8111-111111111111'
    )

    const insertCall = pgQuery.mock.calls.find(([sql]) =>
      String(sql).includes('INSERT INTO workflow_schedules')
    )
    expect(insertCall).toBeDefined()
    expect(String(insertCall?.[0])).toContain('concurrency_policy')
    expect((insertCall?.[1] as unknown[] | undefined)?.[10]).toBe('Allow')
  })

  it('defaults concurrency_policy to Forbid when triggers.schedule omits it', async () => {
    const { pool: pgPool, query: pgQuery } = makePgPool()
    const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))

    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec({ triggers: { schedule: { cron: '*/5 * * * *', suspend: true } } }),
      undefined,
      undefined,
      undefined,
      undefined,
      '11111111-1111-4111-8111-111111111111'
    )

    const insertCall = pgQuery.mock.calls.find(([sql]) =>
      String(sql).includes('INSERT INTO workflow_schedules')
    )
    expect((insertCall?.[1] as unknown[] | undefined)?.[10]).toBe('Forbid')
    // suspend still maps to enabled = false alongside the policy.
    expect((insertCall?.[1] as unknown[] | undefined)?.[6]).toBe(false)
  })

  it('calls tokenFactory to generate coordinator tokens (WRC→mcp-host tokens are now signed per-request, not on reconcile)', async () => {
    const reconciler = new WorkflowReconciler(deps)
    await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    const tf = deps.tokenFactory as unknown as ReturnType<typeof makeTokenFactory>
    // Coordinator tokens are persistent (Secret-mounted) → signed at reconcile time.
    expect(tf.signCoordinatorToMcpHostToken).toHaveBeenCalled()
    expect(tf.signCoordinatorToWrcToken).toHaveBeenCalled()
    // WRC→mcp-host configure/artifact tokens are no longer issued on reconcile —
    // they are signed fresh per call by restEndpoints handlers.
    expect(tf.signWrcConfigureToken).not.toHaveBeenCalled()
    expect(tf.signWrcArtifactToken).not.toHaveBeenCalled()
  })

  it('is idempotent — Pod already running causes no crash (readNamespacedPod returns existing)', async () => {
    const coreApiWithExistingPod = makeCoreApi(true)
    const idempotentDeps = makeDeps({ coreApi: coreApiWithExistingPod as never })
    const reconciler = new WorkflowReconciler(idempotentDeps)

    // Should not throw on second reconcile
    const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(result.phase).toBeDefined()
  })

  it('deletes stale mcp-host and artifact reader before coordinator recovery and recreates runtime pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedPod = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'test-wf-coordinator') {
        return {
          status: { phase: 'Failed' },
          spec: { containers: [{ image: 'clerum/coordinator:test' }] },
        }
      }
      if (name === 'test-wf-mcp-host') {
        return {
          status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }] },
          spec: { containers: [{ image: 'mcp-host' }] },
        }
      }
      if (name === 'test-wf-workflow-output-anchor') {
        return { status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }] } }
      }
      if (name === 'test-wf-workflow-output-prepare') {
        return { status: { phase: 'Succeeded' } }
      }
      throw { code: 404 }
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      {
        workflowExecution: { phase: 'running', attempt: 0 },
      },
      undefined,
      undefined,
      runId
    )

    expect(result.phase).toBeDefined()
    const deletedNames = coreApi.deleteNamespacedPod.mock.calls.map(([arg]) => arg.name)
    expect(deletedNames.slice(0, 3)).toEqual([
      'test-wf-mcp-host',
      'test-wf-artifact-reader',
      'test-wf-coordinator',
    ])
    const createdPodNames = coreApi.createNamespacedPod.mock.calls.map(
      ([arg]) => arg.body?.metadata?.name
    )
    expect(createdPodNames).toContain('test-wf-mcp-host')
    expect(createdPodNames).toContain('test-wf-artifact-reader')
    expect(createdPodNames).toContain('test-wf-coordinator')
  })

  it('handles spec with no mcpServers (pure-compute workflow)', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const spec = makeSpec({ mcpServers: undefined })
    const result = await reconciler.reconcile('pure-compute', 'uid-pc', 'sandbox-recipes', spec)
    expect(result.phase).toBeDefined()
    // Should not crash on missing mcpServers
  })

  it('returns failed when coreApi createNamespacedSecret rejects with non-404', async () => {
    const errorDeps = makeDeps({
      coreApi: {
        ...makeCoreApi(),
        createNamespacedSecret: vi.fn().mockRejectedValue(new Error('quota exceeded')),
      } as never,
    })
    const reconciler = new WorkflowReconciler(errorDeps)
    const result = await reconciler.reconcile('test-wf', 'uid-123', 'sandbox-recipes', makeSpec())
    expect(result.phase).toBe('failed')
    expect(result.message).toBeDefined()
  })

  it('fails reconcile when creating the mcpHost runtime token Secret fails', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-mcp-host-runtime-tokens') {
        throw new Error('approval secret create failed')
      }
      return {}
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).toBe('failed')
    expect(result.message).toContain('approval secret create failed')
  })

  it('refreshes the mcpHost runtime token Secret when stored tokens are near expiry', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(makeJwtWithExp(30)).toString('base64'),
          },
        }
      }
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-mcp-host-runtime-tokens',
        namespace: 'sandbox-recipes',
      }),
      expect.any(Object)
    )
  })

  it('uses the configured runtime token refresh threshold for mcpHost runtime tokens', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(
              makeJwtWithExp(1200, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(
              makeJwtWithExp(1200, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(
              makeJwtWithExp(1200, { recipeName: 'test-wf' })
            ).toString('base64'),
          },
        }
      }
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        config: {
          ...makeConfig(),
          runtimeTokenTtlSeconds: 3600,
          runtimeTokenRefreshBeforeSeconds: 1800,
        } as never,
      })
    )
    await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({ name: 'wf-test-wf-mcp-host-runtime-tokens' }),
      expect.any(Object)
    )
  })

  it('reissues mcpHost runtime tokens when the stored recipeName no longer matches the runtime scope', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-child-run-mcp-host-runtime-tokens') {
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'child-run' })
            ).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'child-run' })
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'child-run' })
            ).toString('base64'),
            'mcp-host-gfs-token': makeEncodedGfsAccess(3600, 'child-run'),
          },
        }
      }
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'child-run',
      makeSpec(),
      'parent-recipe'
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'parent-recipe', [])
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-child-run-mcp-host-runtime-tokens',
        namespace: 'sandbox-recipes',
        body: expect.objectContaining({
          data: {
            'mcp-host-runtime-access-token': Buffer.from('mcp-host-runtime-access-token').toString(
              'base64'
            ),
            'mcp-host-runtime-refresh-token': Buffer.from(
              'mcp-host-runtime-refresh-token'
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(
              'mcp-host-workflow-control-token'
            ).toString('base64'),
            'mcp-host-gfs-token': Buffer.from('gfs-runtime-value').toString('base64'),
          },
          metadata: expect.objectContaining({
            annotations: expect.objectContaining({
              'clerum.io/mcp-host-runtime-token-generation': '1',
            }),
          }),
        }),
      }),
      expect.any(Object)
    )
  })

  it('treats mcpHost runtime token Secret 409 as idempotent and reuses the existing Secret', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    let mcpHostRuntimeSecretCreated = false

    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
        if (!mcpHostRuntimeSecretCreated) throw { code: 404 }
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(makeJwtWithExp(3600)).toString('base64'),
            'mcp-host-gfs-token': makeEncodedGfsAccess(),
          },
        }
      }
      throw { code: 404 }
    })
    coreApi.createNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      if (body?.metadata?.name === 'wf-test-wf-mcp-host-runtime-tokens') {
        mcpHostRuntimeSecretCreated = true
        throw { code: 409 }
      }
      return {}
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).not.toBe('failed')
    expect(coreApi.createNamespacedSecret).toHaveBeenCalled()
    expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
  })

  it('keeps the existing mcpHost runtime token Secret when stored tokens are healthy', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf' })
            ).toString('base64'),
            'mcp-host-gfs-token': makeEncodedGfsAccess(),
          },
        }
      }
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    const callsBefore = vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length
    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'recipe-runtime-ns',
      makeSpec(),
      undefined,
      undefined,
      undefined,
      runId
    )

    expect(result.phase).not.toBe('failed')
    expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length).toBe(callsBefore)
    expect(coreApi.patchNamespacedSecret).not.toHaveBeenCalled()
  })

  it('reissues stale access and refresh scopes once, then converges on the next reconcile', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const expectedWorkflowControlScopes = [
      'workflow:trigger',
      'workflow:approval:resolve',
      'workflow:approval:decide',
    ]
    const canonicalAccessToken = makeJwtWithExp(3600, {
      workflowControlScopes: expectedWorkflowControlScopes,
    })
    const canonicalRefreshToken = makeJwtWithExp(3600, {
      workflowControlScopes: expectedWorkflowControlScopes,
    })
    const canonicalControlToken = makeJwtWithExp(3600, {
      scopes: expectedWorkflowControlScopes,
    })
    vi.mocked(issueMcpHostRuntimeTokens).mockResolvedValue({
      accessToken: canonicalAccessToken,
      refreshToken: canonicalRefreshToken,
      mcpHostControlToken: canonicalControlToken,
    })

    let secretData: Record<string, string> = {
      'mcp-host-runtime-access-token': Buffer.from(
        makeJwtWithExp(3600, { workflowControlScopes: ['workflow:list'] })
      ).toString('base64'),
      'mcp-host-runtime-refresh-token': Buffer.from(
        makeJwtWithExp(3600, { workflowControlScopes: ['workflow:list'] })
      ).toString('base64'),
      'mcp-host-workflow-control-token': Buffer.from(
        makeJwtWithExp(3600, { scopes: expectedWorkflowControlScopes })
      ).toString('base64'),
      'mcp-host-gfs-token': makeEncodedGfsAccess(),
    }
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') return { data: secretData }
      throw { code: 404 }
    })
    coreApi.patchNamespacedSecret = vi.fn().mockImplementation(async ({ body }) => {
      secretData = { ...secretData, ...(body?.data ?? {}) }
      return {}
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    const spec = makeSpec({
      steps: [
        {
          id: 'broker-trigger',
          instruction: 'Trigger a workflow',
          allowedTools: { include: ['workflow_trigger'] },
        },
      ],
    })

    await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', spec)
    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledTimes(1)
    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
      'recipe-runtime-ns',
      'test-wf',
      expectedWorkflowControlScopes
    )
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)

    await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', spec)
    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledTimes(1)
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledTimes(1)
  })

  it('reissues only the workflow-control token when declared broker scopes change', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const expectedWorkflowControlScopes = [
      'workflow:trigger',
      'workflow:approval:resolve',
      'workflow:approval:decide',
    ]
    coreApi.readNamespacedSecret = vi.fn().mockImplementation(async ({ name }) => {
      if (name === 'wf-test-wf-mcp-host-runtime-tokens') {
        return {
          data: {
            'mcp-host-runtime-access-token': Buffer.from(
              makeJwtWithExp(3600, {
                recipeName: 'test-wf',
                workflowControlScopes: expectedWorkflowControlScopes,
              })
            ).toString('base64'),
            'mcp-host-runtime-refresh-token': Buffer.from(
              makeJwtWithExp(3600, {
                recipeName: 'test-wf',
                workflowControlScopes: expectedWorkflowControlScopes,
              })
            ).toString('base64'),
            'mcp-host-workflow-control-token': Buffer.from(
              makeJwtWithExp(3600, { recipeName: 'test-wf', scopes: ['workflow:list'] })
            ).toString('base64'),
            'mcp-host-gfs-token': makeEncodedGfsAccess(),
          },
        }
      }
      throw { code: 404 }
    })

    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))
    const runtimeCallsBefore = vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length
    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({
        steps: [
          {
            id: 'broker-trigger',
            instruction: 'Trigger a workflow',
            allowedTools: { include: ['workflow_trigger'] },
          },
        ],
      })
    )

    expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.length).toBe(runtimeCallsBefore)
    expect(issueMcpHostWorkflowControlToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
      ...expectedWorkflowControlScopes,
    ])
    expect(coreApi.patchNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        name: 'wf-test-wf-mcp-host-runtime-tokens',
        namespace: 'sandbox-recipes',
        body: expect.objectContaining({
          data: {
            'mcp-host-workflow-control-token': Buffer.from(
              'mcp-host-workflow-control-token'
            ).toString('base64'),
          },
          metadata: expect.objectContaining({
            annotations: expect.objectContaining({
              'clerum.io/mcp-host-runtime-token-generation': '1',
            }),
          }),
        }),
      }),
      expect.any(Object)
    )
  })

  it('can repair a missing mcpHost runtime token Secret without recreating workflow pods', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureMcpHostRuntimeCredentials('recipe-runtime-ns', 'test-wf', makeSpec())

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [])
    expect(coreApi.createNamespacedSecret).toHaveBeenCalledWith(
      expect.objectContaining({
        namespace: 'sandbox-recipes',
        body: expect.objectContaining({
          metadata: expect.objectContaining({
            name: 'wf-test-wf-mcp-host-runtime-tokens',
          }),
        }),
      })
    )
    expect(coreApi.createNamespacedPod).not.toHaveBeenCalled()
  })

  it('derives workflow-control scopes from declared workflow broker tools', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({
        steps: [
          {
            id: 'broker',
            instruction: 'Use workflow broker',
            allowedTools: {
              include: [
                'clerum__list_workflows',
                'clerum__read_workflow',
                'clerum__trigger_workflow',
              ],
            },
          },
        ],
      })
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
      'workflow:list',
      'workflow:read',
      'workflow:trigger',
      'workflow:approval:resolve',
      'workflow:approval:decide',
    ])
  })

  it('derives write-scoped GFS host tokens from declared publish targets', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({
        gfs: {
          publishTargets: [{ drive: 'main', target: 'outputs' }],
        },
      })
    )

    expect(mintRecipeHostGfsToken).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', {
      scopes: ['gfs.read', 'gfs.write'],
    })
  })

  it('derives workflow approval scopes from agentic approval-gated steps', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({
        agent: { provider: 'zai', model: 'glm-4.7' },
        steps: [
          {
            id: 'approval-gated',
            instruction: 'Continue after approval',
            requiresApproval: {
              target: { userId: 'user-1' },
              message: 'Approve this step',
            },
          },
        ],
      })
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith('recipe-runtime-ns', 'test-wf', [
      'workflow:approval:resolve',
      'workflow:approval:decide',
    ])
  })

  it('does not create mcpHost runtime credentials for snippet-only workflows', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.ensureMcpHostRuntimeCredentials(
      'recipe-runtime-ns',
      'test-wf',
      makeSpec({ steps: [{ id: 's1', run: snippetRun() }] })
    )

    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
    expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
  })

  it('cleans workflow output through artifact-reader before deleting runtime resources', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedService = vi.fn().mockImplementation(({ name }: { name: string }) => {
      if (name === 'wf-test-wf-artifact-reader') return Promise.resolve({})
      throw { code: 404 }
    })
    const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 204 } as Response)
    vi.stubGlobal('fetch', fetchMock)
    const tokenFactory = makeTokenFactory()
    const reconciler = new WorkflowReconciler(
      makeDeps({ coreApi: coreApi as never, tokenFactory: tokenFactory as never })
    )

    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')

    expect(tokenFactory.signWrcArtifactDeleteToken).toHaveBeenCalledWith(
      'test-wf',
      'sandbox-recipes'
    )
    expect(fetchMock).toHaveBeenCalledWith(
      'http://wf-test-wf-artifact-reader.sandbox-recipes.svc.cluster.local:8080/api/v1/workflow/artifacts',
      expect.objectContaining({
        method: 'DELETE',
        headers: { authorization: 'Bearer wrc-artifact-delete-token' },
      })
    )
    expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith(
      expect.objectContaining({ name: 'test-wf-artifact-reader' })
    )
  })

  it('keeps the finalizer retryable when runtime pod cleanup fails', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.deleteNamespacedPod = vi.fn().mockImplementation(({ name }: { name: string }) => {
      if (name === 'test-wf-workflow-output-anchor') {
        return Promise.reject({ code: 500, message: 'api server unavailable' })
      }
      return Promise.resolve({})
    })
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await expect(reconciler.reconcileDelete('test-wf', 'sandbox-recipes')).rejects.toThrow(
      'test-wf-workflow-output-anchor'
    )

    expect(coreApi.deleteNamespacedPod).toHaveBeenCalledWith(
      expect.objectContaining({ name: 'test-wf-coordinator' })
    )
    expect(coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      name: 'test-wf-workflow-output',
      namespace: 'sandbox-recipes',
    })
  })

  it('falls back to mcp-host artifact cleanup when artifact-reader is unavailable', async () => {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedService = vi.fn().mockImplementation(({ name }: { name: string }) => {
      if (name === 'wf-test-wf-mcp-host') return Promise.resolve({})
      throw { code: 404 }
    })
    const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 204 } as Response)
    vi.stubGlobal('fetch', fetchMock)
    const reconciler = new WorkflowReconciler(makeDeps({ coreApi: coreApi as never }))

    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')

    expect(fetchMock).toHaveBeenCalledTimes(1)
    expect(String(fetchMock.mock.calls[0][0])).toBe(
      'http://wf-test-wf-mcp-host.sandbox-recipes.svc.cluster.local:8080/api/v1/workflow/artifacts'
    )
  })

  it('deletes workflow trigger policy and cancels live approvals for the deleted recipe', async () => {
    const { pool: pgPool, query, release } = makePgPool()
    query.mockImplementation(async (sql: string) => {
      if (String(sql).includes("to_regclass('public.workflow_recipe_allowed_users')")) {
        return { rows: [{ regclass: 'workflow_recipe_allowed_users' }], rowCount: 1 }
      }
      const trimmed = String(sql).trim().toUpperCase()
      if (trimmed.startsWith('SELECT')) {
        return { rows: [], rowCount: 0 }
      }
      return { rows: [], rowCount: 1 }
    })
    const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))

    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')

    const sqls = query.mock.calls.map(([sql]) => String(sql))
    expect(sqls).toContainEqual('BEGIN')
    expect(sqls).toContainEqual(expect.stringContaining('DELETE FROM user_workflow_triggers'))
    expect(sqls).toContainEqual(expect.stringContaining('DELETE FROM team_workflow_triggers'))
    expect(sqls).toContainEqual(
      expect.stringContaining('DELETE FROM workflow_recipe_allowed_teams')
    )
    expect(sqls).toContainEqual(
      expect.stringContaining("to_regclass('public.workflow_recipe_allowed_users')")
    )
    expect(sqls).toContainEqual(
      expect.stringContaining('DELETE FROM workflow_recipe_allowed_users')
    )
    expect(sqls).toContainEqual(expect.stringContaining('UPDATE workflow_approval_requests'))
    expect(sqls).toContainEqual(expect.stringContaining("status IN ('pending', 'approved')"))
    expect(sqls).toContainEqual(
      expect.stringContaining("cancelled_by = COALESCE(cancelled_by, 'workflow-recipe-delete')")
    )
    expect(sqls).toContainEqual('COMMIT')
    expect(query.mock.calls).toContainEqual([
      expect.stringContaining('DELETE FROM team_workflow_triggers'),
      ['sandbox-recipes', 'test-wf'],
    ])
    expect(query.mock.calls).toContainEqual([
      expect.stringContaining('DELETE FROM workflow_recipe_allowed_teams'),
      ['sandbox-recipes', 'test-wf'],
    ])
    expect(query.mock.calls).toContainEqual([
      expect.stringContaining('DELETE FROM workflow_recipe_allowed_users'),
      ['sandbox-recipes', 'test-wf'],
    ])
    expect(query.mock.calls).toContainEqual([
      expect.stringContaining('UPDATE workflow_approval_requests'),
      ['sandbox-recipes', 'test-wf'],
    ])
    expect(release).toHaveBeenCalledTimes(1)
  })

  it('keeps recipe-delete cleanup idempotent when legacy allowed-users table is absent', async () => {
    const { pool: pgPool, query, release } = makePgPool()
    const reconciler = new WorkflowReconciler(makeDeps({ pgPool }))

    await reconciler.reconcileDelete('test-wf', 'sandbox-recipes')

    const sqls = query.mock.calls.map(([sql]) => String(sql))
    expect(sqls).toContainEqual(
      expect.stringContaining("to_regclass('public.workflow_recipe_allowed_users')")
    )
    expect(sqls).not.toContainEqual(
      expect.stringContaining('DELETE FROM workflow_recipe_allowed_users')
    )
    expect(sqls).toContainEqual('COMMIT')
    expect(release).toHaveBeenCalledTimes(1)
  })

  it('workflowPhase in result is a valid WorkflowPhase value', async () => {
    const reconciler = new WorkflowReconciler(deps)
    const result = await reconciler.reconcile(
      'test-wf',
      'uid-123',
      'sandbox-recipes',
      makeSpec(),
      undefined,
      undefined,
      'test-wf',
      runId
    )
    const validPhases = [
      'pending',
      'approved',
      'deploying',
      'running',
      'awaiting_approval',
      'completed',
      'failed',
      'cancelling',
      'cancelled',
      'recovering',
      'initializing',
    ]
    expect(validPhases).toContain(result.workflowPhase)
  })
})

describe('cleanupPluginWorkloadSdk legacy internet policy ownership', () => {
  const RECIPE = 'sdk-only'
  const RECIPE_UID = 'uid-sdk-only'
  const LEGACY = `${RECIPE}-mcp-servers-egress-internet`

  function legacyDeletes(networkingApi: ReturnType<typeof makeNetworkingApi>): number {
    return networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
      ([arg]) => arg.name === LEGACY
    ).length
  }

  function makeCleanupCoreApi(): ReturnType<typeof makeCoreApi> {
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
    return coreApi
  }

  it('routes a labeled legacy policy through the shared live ledger instead of the label sweep', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    let legacyPresent = true
    networkingApi.listNamespacedNetworkPolicy.mockImplementation(
      async ({ namespace }: { namespace: string }) => ({
        items:
          namespace === 'mcp-server'
            ? legacyPresent
              ? [
                  {
                    metadata: {
                      name: LEGACY,
                      namespace,
                      labels: { 'clerum.io/recipe': RECIPE, 'clerum.io/managed-by': 'wrc' },
                    },
                  },
                ]
              : []
            : [],
      })
    )
    networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
      if (name === LEGACY) legacyPresent = false
      return {}
    })
    const coreApi = makeCoreApi() as ReturnType<typeof makeCoreApi>
    coreApi.readNamespacedEndpoints = vi.fn().mockRejectedValue({ code: 404 })
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: coreApi as never,
        networkingApi: networkingApi as never,
        pluginWorkloadSdkRevocationClient: {
          revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
        } as never,
      })
    )

    await reconciler.cleanupPluginWorkloadSdk(RECIPE, { recipeUid: RECIPE_UID })

    // Liveness witness: both label-sweep namespaces were evaluated, and the
    // shared by-name path sent the sole legacy DELETE.
    expect(networkingApi.listNamespacedNetworkPolicy).toHaveBeenCalledWith(
      expect.objectContaining({
        namespace: 'mcp-server',
        labelSelector: `clerum.io/recipe=${RECIPE},clerum.io/managed-by=wrc`,
      })
    )
    expect(legacyDeletes(networkingApi)).toBe(1)

    await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
    expect(legacyDeletes(networkingApi)).toBe(1)
  })

  it('keeps live cleanup on the shared backoff while finalizer cleanup gets one last chance', async () => {
    vi.useFakeTimers({ toFake: ['Date'] })
    try {
      const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
      let legacyDeleteFails = true
      networkingApi.deleteNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
        if (name === LEGACY && legacyDeleteFails) throw { code: 403, message: 'forbidden' }
        return {}
      })
      const reconciler = new WorkflowReconciler(
        makeDeps({
          coreApi: makeCleanupCoreApi() as never,
          networkingApi: networkingApi as never,
          pluginWorkloadSdkRevocationClient: {
            revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
          } as never,
        })
      )

      await expect(
        reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
      ).resolves.toBe('pending')
      expect(legacyDeletes(networkingApi)).toBe(1)

      const pending = await reconciler
        .cleanupPluginWorkloadSdk(RECIPE, {
          recipeUid: RECIPE_UID,
        })
        .catch(error => error)
      expect(pending).toBeInstanceOf(LegacyNetworkPolicyDeletePendingError)
      expect((pending as LegacyNetworkPolicyDeletePendingError).retryAfterMs).toBe(
        LEGACY_NETWORK_POLICY_DELETE_BACKOFF_BASE_MS
      )
      expect(legacyDeletes(networkingApi)).toBe(1)

      legacyDeleteFails = false
      await reconciler.cleanupPluginWorkloadSdk(RECIPE, {
        recipeUid: RECIPE_UID,
        recipeDeleted: true,
      })
      expect(legacyDeletes(networkingApi)).toBe(2)

      await reconciler.retryLegacyMcpServersInternetEgressDelete(RECIPE, RECIPE_UID)
      expect(legacyDeletes(networkingApi)).toBe(3)
    } finally {
      vi.useRealTimers()
    }
  })

  it('keeps the legacy policy live when cleanup preserves the workflow runtime', async () => {
    const networkingApi = makeNetworkingApi() as ReturnType<typeof makeNetworkingApi>
    const reconciler = new WorkflowReconciler(
      makeDeps({
        coreApi: makeCleanupCoreApi() as never,
        networkingApi: networkingApi as never,
        pluginWorkloadSdkRevocationClient: {
          revoke: vi.fn().mockResolvedValue({ state: 'disabled' }),
        } as never,
      })
    )

    await reconciler.cleanupPluginWorkloadSdk('hybrid', {
      recipeUid: 'uid-hybrid',
      preserveWorkflowRuntime: true,
    })

    expect(
      networkingApi.deleteNamespacedNetworkPolicy.mock.calls.filter(
        ([arg]) => arg.name === 'hybrid-mcp-servers-egress-internet'
      )
    ).toHaveLength(0)
    expect(networkingApi.listNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })
})

describe('buildNetworkPolicyOwnershipConditions', () => {
  it('names the conflicting policies in sorted order, whatever order the apply produced them in', () => {
    const conditions = buildNetworkPolicyOwnershipConditions(
      {
        conflicts: [
          { policy: 'test-wf-wrc-to-artifact-reader', reason: 'identity-label-mismatch' },
          { policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' },
        ],
        retryPending: false,
      },
      '2026-09-23T10:00:00.000Z'
    )

    expect(conditions).toEqual([
      {
        type: 'WorkflowNetworkPolicyOwnership',
        status: 'False',
        reason: 'OwnershipConflict',
        message:
          'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch), test-wf-wrc-to-artifact-reader (identity-label-mismatch)',
        lastTransitionTime: '2026-09-23T10:00:00.000Z',
      },
    ])
  })
})

// The one translation of a full pass summary into status fields, shared by
// the run lane, the eager lane and the SDK-only lane (the `reconcile` writer
// of Table A). `toStrictEqual` pins which keys are present: an absent
// ownership key keeps the published condition, so it must not be emitted as
// an explicit `undefined`.
describe('translateNetworkPolicyPassSummary', () => {
  const now = '2026-09-23T12:00:00.000Z'
  const earlier = '2026-09-20T08:00:00.000Z'
  const pass = (overrides: Partial<NetworkPolicyPassSummary> = {}): NetworkPolicyPassSummary => ({
    conflicts: [],
    retryPending: false,
    prune: 'converged',
    legacy: 'removed',
    ...overrides,
  })
  const conflictSummary = pass({
    conflicts: [{ policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' as const }],
  })
  const publishedConflict = {
    type: 'WorkflowNetworkPolicyOwnership',
    status: 'False' as const,
    reason: 'OwnershipConflict',
    message: 'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
    lastTransitionTime: earlier,
  }
  const publishedRetryMarker = {
    type: 'WorkflowNetworkPoliciesConverged',
    status: 'False' as const,
    reason: 'RetryPending',
    message: 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
    lastTransitionTime: earlier,
  }

  it('emits neither field when the pass produced no summary, so the published condition is kept', () => {
    const result = translateNetworkPolicyPassSummary(undefined, [publishedConflict], now)

    expect(result).toStrictEqual({})
    // Witness: the same existing conditions with a summary do produce the
    // field, so the empty result above is caused by the missing summary.
    expect(
      translateNetworkPolicyPassSummary(conflictSummary, [publishedConflict], now)
        .networkPolicyOwnershipConditions
    ).toEqual([publishedConflict])
  })

  it('clears the condition with [] and sets no retry flag when nothing conflicts or is pending', () => {
    const result = translateNetworkPolicyPassSummary(pass(), [publishedConflict], now)

    expect(result.networkPolicyOwnershipConditions).toEqual([])
    expect(result).not.toHaveProperty('networkPolicyRetryPending')
    expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
  })

  it('replaces the conflict with the retry marker and sets the retry flag when a retry is pending without a conflict', () => {
    const result = translateNetworkPolicyPassSummary(
      pass({ retryPending: true }),
      [publishedConflict],
      now
    )

    expect(result).toStrictEqual({
      networkPolicyOwnershipConditions: [{ ...publishedRetryMarker, lastTransitionTime: now }],
      networkPolicyRetryPending: true,
    })
  })

  it('keeps the lastTransitionTime of a published retry marker while the retry stays pending', () => {
    const result = translateNetworkPolicyPassSummary(
      pass({ retryPending: true }),
      [publishedRetryMarker],
      now
    )

    expect(result).toStrictEqual({
      networkPolicyOwnershipConditions: [publishedRetryMarker],
      networkPolicyRetryPending: true,
    })
  })

  it('clears a published retry marker with [] once no retry is pending', () => {
    const result = translateNetworkPolicyPassSummary(pass(), [publishedRetryMarker], now)

    expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
  })

  it('replaces the condition with one False condition naming every conflicting policy', () => {
    const result = translateNetworkPolicyPassSummary(
      pass({
        conflicts: [
          { policy: 'test-wf-wrc-to-artifact-reader', reason: 'identity-label-mismatch' },
          { policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' },
        ],
      }),
      undefined,
      now
    )

    expect(result.networkPolicyOwnershipConditions).toEqual([
      {
        type: 'WorkflowNetworkPolicyOwnership',
        status: 'False',
        reason: 'OwnershipConflict',
        message:
          'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch), test-wf-wrc-to-artifact-reader (identity-label-mismatch)',
        lastTransitionTime: now,
      },
    ])
    expect(result).not.toHaveProperty('networkPolicyRetryPending')
  })

  it('keeps the lastTransitionTime of an identical published condition', () => {
    const result = translateNetworkPolicyPassSummary(conflictSummary, [publishedConflict], now)

    expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [publishedConflict] })
    expect(result.networkPolicyOwnershipConditions?.[0]?.lastTransitionTime).toBe(earlier)
  })

  it('reports a conflict and a pending retry from the same pass together', () => {
    const result = translateNetworkPolicyPassSummary(
      { ...conflictSummary, retryPending: true },
      undefined,
      now
    )

    expect(result).toStrictEqual({
      networkPolicyOwnershipConditions: [
        { ...publishedConflict, lastTransitionTime: now },
        { ...publishedRetryMarker, lastTransitionTime: now },
      ],
      networkPolicyRetryPending: true,
    })
  })

  // R3-L3: a prune that could not delete (a non-404 DELETE) is its own fact;
  // a failed LIST is `unevaluated` (R4-L6). It is published as PrunePending
  // and never sets the retry flag: nothing in the run's short-circuits can
  // prune, so a requeue would only repeat the failed DELETE.
  describe('prune-pending', () => {
    const publishedPruneMarker = {
      type: 'WorkflowNetworkPoliciesConverged',
      status: 'False' as const,
      reason: 'PrunePending',
      message:
        'One or more run-lane NetworkPolicies the spec no longer wants are pending a delete; the next reconcile() pass or the finalizer removes them',
      lastTransitionTime: earlier,
    }

    it('publishes PrunePending without the retry flag when only the prune is pending', () => {
      const result = translateNetworkPolicyPassSummary(
        pass({ prune: 'pending' }),
        [publishedConflict],
        now
      )

      expect(result).toStrictEqual({
        networkPolicyOwnershipConditions: [{ ...publishedPruneMarker, lastTransitionTime: now }],
      })
    })

    it('clears a published PrunePending marker once a pass pruned', () => {
      const result = translateNetworkPolicyPassSummary(pass(), [publishedPruneMarker], now)

      expect(result).toStrictEqual({ networkPolicyOwnershipConditions: [] })
    })

    it('reports a pending retry and a pending prune in one RetryAndPrunePending marker', () => {
      const result = translateNetworkPolicyPassSummary(
        pass({ retryPending: true, prune: 'pending' }),
        [publishedPruneMarker],
        now
      )

      expect(result).toStrictEqual({
        networkPolicyOwnershipConditions: [
          {
            type: 'WorkflowNetworkPoliciesConverged',
            status: 'False',
            reason: 'RetryAndPrunePending',
            message:
              'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
            lastTransitionTime: earlier,
          },
        ],
        networkPolicyRetryPending: true,
      })
    })
  })

  // R4-L6: a failed prune LIST evaluated nothing, so the prune fact is
  // `unevaluated`, never `pending`. It has its own reasons and sets no retry
  // flag of its own.
  describe('prune-unevaluated', () => {
    const convergedMarkers = (result: ReturnType<typeof translateNetworkPolicyPassSummary>) =>
      (result.networkPolicyOwnershipConditions ?? []).filter(
        c => c.type === 'WorkflowNetworkPoliciesConverged'
      )

    it('R4-L6: publishes PruneUnevaluated without the retry flag when the prune LIST failed', () => {
      const result = translateNetworkPolicyPassSummary(
        pass({ prune: 'unevaluated' }),
        [publishedConflict],
        now
      )

      expect(convergedMarkers(result)).toMatchObject([
        { status: 'False', reason: 'PruneUnevaluated', lastTransitionTime: now },
      ])
      expect(result.networkPolicyRetryPending).toBeUndefined()
    })

    it('R4-L6: publishes RetryAndPruneUnevaluated for a pending retry and an unevaluated prune', () => {
      const result = translateNetworkPolicyPassSummary(
        pass({ retryPending: true, prune: 'unevaluated' }),
        [],
        now
      )

      expect(convergedMarkers(result)).toMatchObject([
        { status: 'False', reason: 'RetryAndPruneUnevaluated' },
      ])
      expect(result.networkPolicyRetryPending).toBe(true)
    })
  })

  // R4-L3: the legacy policy is its own fact, on its own condition.
  describe('legacy', () => {
    const legacyMarkers = (result: ReturnType<typeof translateNetworkPolicyPassSummary>) =>
      (result.networkPolicyOwnershipConditions ?? []).filter(
        c => c.type === 'WorkflowLegacyNetworkPolicyRemoved'
      )

    it('R4-L3: publishes DeletePending on its own condition, apart from the prune marker', () => {
      const result = translateNetworkPolicyPassSummary(pass({ legacy: 'pending' }), [], now)

      expect(result).toStrictEqual({
        networkPolicyOwnershipConditions: [
          {
            type: 'WorkflowLegacyNetworkPolicyRemoved',
            status: 'False',
            reason: 'DeletePending',
            message:
              'The legacy mcp-servers internet egress NetworkPolicy is pending a delete; a later pass or the finalizer removes it',
            lastTransitionTime: now,
          },
        ],
      })
    })

    it('R4-L3: keeps the first transition time while the delete stays pending, and clears it once removed', () => {
      const published = translateNetworkPolicyPassSummary(
        pass({ legacy: 'pending' }),
        [],
        earlier
      ).networkPolicyOwnershipConditions!
      // Liveness witness: the failing pass published the marker.
      expect(legacyMarkers({ networkPolicyOwnershipConditions: published })).toHaveLength(1)

      const still = translateNetworkPolicyPassSummary(pass({ legacy: 'pending' }), published, now)
      expect(legacyMarkers(still)).toMatchObject([{ lastTransitionTime: earlier }])

      expect(translateNetworkPolicyPassSummary(pass(), published, now)).toStrictEqual({
        networkPolicyOwnershipConditions: [],
      })
    })
  })
})

// R4-L3: every writer goes through `networkPolicyMarkerConditions`, and each
// fact is observed, carried or cleared as Table A says.
describe('networkPolicyMarkerConditions', () => {
  const now = '2026-09-23T12:00:00.000Z'
  const earlier = '2026-09-20T08:00:00.000Z'
  const pass = (overrides: Partial<NetworkPolicyPassSummary> = {}): NetworkPolicyPassSummary => ({
    conflicts: [],
    retryPending: false,
    prune: 'converged',
    legacy: 'removed',
    ...overrides,
  })
  const conflict = { policy: 'test-wf-coord-to-wrc', reason: 'owner-reference-mismatch' as const }
  const published = (overrides: Partial<NetworkPolicyPassSummary>) =>
    networkPolicyMarkerConditions({ kind: 'reconcile', summary: pass(overrides) }, [], earlier)
  const reasons = (conditions: { type: string; reason?: string }[]) =>
    conditions.map(c => `${c.type}/${c.reason}`)

  describe('apply-retry', () => {
    it('R4-L3: observes the apply and carries the prune and legacy facts', () => {
      const existing = published({ retryPending: true, prune: 'pending', legacy: 'pending' })
      // Liveness witness: all three facts were published.
      expect(reasons(existing)).toEqual([
        'WorkflowNetworkPoliciesConverged/RetryAndPrunePending',
        'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
      ])

      const settled = networkPolicyMarkerConditions(
        { kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },
        existing,
        now
      )

      expect(reasons(settled)).toEqual([
        'WorkflowNetworkPoliciesConverged/PrunePending',
        'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
      ])
      expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)
    })

    it('R4-L3: carries an unevaluated prune through a pending retry', () => {
      const existing = published({ prune: 'unevaluated' })

      const retried = networkPolicyMarkerConditions(
        { kind: 'apply-retry', summary: { conflicts: [], retryPending: true } },
        existing,
        now
      )

      expect(reasons(retried)).toEqual([
        'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',
      ])
    })

    it('R4-L3: publishes the conflicts the retry observed', () => {
      const retried = networkPolicyMarkerConditions(
        { kind: 'apply-retry', summary: { conflicts: [conflict], retryPending: false } },
        [],
        now
      )

      expect(reasons(retried)).toEqual(['WorkflowNetworkPolicyOwnership/OwnershipConflict'])
    })
  })

  describe('legacy-retry', () => {
    it('R4-L4: clears the legacy fact once removed and carries the apply, the prune and the ownership condition', () => {
      const existing = published({
        conflicts: [conflict],
        retryPending: true,
        prune: 'unevaluated',
        legacy: 'pending',
      })
      // Liveness witness: all three conditions were published.
      expect(existing).toHaveLength(3)

      const settled = networkPolicyMarkerConditions(
        { kind: 'legacy-retry', legacy: 'removed' },
        existing,
        now
      )

      expect(reasons(settled)).toEqual([
        'WorkflowNetworkPolicyOwnership/OwnershipConflict',
        'WorkflowNetworkPoliciesConverged/RetryAndPruneUnevaluated',
      ])
      expect(settled.every(c => c.lastTransitionTime === earlier)).toBe(true)
    })

    it('R4-L4: keeps DeletePending and its transition time while the delete stays pending', () => {
      const existing = published({ prune: 'pending', legacy: 'pending' })
      expect(existing).toHaveLength(2)

      expect(
        networkPolicyMarkerConditions({ kind: 'legacy-retry', legacy: 'pending' }, existing, now)
      ).toEqual(existing)
    })

    it('R4-L4: enters only for a published DeletePending', () => {
      expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ legacy: 'pending' }))).toBe(true)
      expect(hasLegacyNetworkPolicyDeletePendingMarker(published({ prune: 'pending' }))).toBe(false)
      expect(hasLegacyNetworkPolicyDeletePendingMarker(undefined)).toBe(false)
      expect(
        hasLegacyNetworkPolicyDeletePendingMarker([
          {
            type: 'WorkflowLegacyNetworkPolicyRemoved',
            status: 'False',
            reason: 'SomethingElse',
            message: 'written by another head',
            lastTransitionTime: earlier,
          },
        ])
      ).toBe(false)
    })
  })

  describe('terminal', () => {
    it('R4-L3: drops a pending apply and carries the prune, the legacy fact and the ownership condition', () => {
      const existing = published({
        conflicts: [conflict],
        retryPending: true,
        prune: 'pending',
        legacy: 'pending',
      })
      // Liveness witness: all three conditions were published.
      expect(existing).toHaveLength(3)

      const settled = networkPolicyMarkerConditions({ kind: 'terminal' }, existing, now)

      expect(reasons(settled)).toEqual([
        'WorkflowNetworkPolicyOwnership/OwnershipConflict',
        'WorkflowNetworkPoliciesConverged/PrunePending',
        'WorkflowLegacyNetworkPolicyRemoved/DeletePending',
      ])
    })

    it('R4-L3: removes a marker that only named a pending apply', () => {
      const existing = published({ retryPending: true })
      expect(reasons(existing)).toEqual(['WorkflowNetworkPoliciesConverged/RetryPending'])

      expect(networkPolicyMarkerConditions({ kind: 'terminal' }, existing, now)).toEqual([])
    })
  })

  describe('unmanaged', () => {
    it('R4-L3: clears every fact after a full SDK teardown', () => {
      const existing = published({
        conflicts: [conflict],
        retryPending: true,
        prune: 'unevaluated',
        legacy: 'pending',
      })
      // Liveness witness: the group was published.
      expect(existing).toHaveLength(3)

      expect(networkPolicyMarkerConditions({ kind: 'unmanaged' }, existing, now)).toEqual([])
    })
  })

  describe('a reason this controller did not write', () => {
    const foreignMarker = {
      type: 'WorkflowNetworkPoliciesConverged',
      status: 'False' as const,
      reason: 'SomethingElse',
      message: 'written by another head',
      lastTransitionTime: earlier,
    }
    const foreignLegacy = {
      type: 'WorkflowLegacyNetworkPolicyRemoved',
      status: 'False' as const,
      reason: 'SomethingElse',
      message: 'written by another head',
      lastTransitionTime: earlier,
    }

    it('R4-L3: is kept as it is by a carrying writer, which warns', () => {
      const warnLog = captureLogger('warn')
      try {
        const carried = networkPolicyMarkerConditions(
          { kind: 'terminal' },
          [foreignMarker, foreignLegacy],
          now
        )

        expect(carried).toEqual([foreignMarker, foreignLegacy])
        expect(warnLog).toHaveBeenCalledTimes(2)
        expect(warnLog).toHaveBeenCalledWith(
          'Keeping a NetworkPolicy marker with a reason this controller did not write',
          {
            writer: 'terminal',
            type: 'WorkflowNetworkPoliciesConverged',
            status: 'False',
            reason: 'SomethingElse',
          }
        )
      } finally {
        warnLog.mockRestore()
      }
    })

    it('R4-L3: is overwritten by reconcile()', () => {
      const warnLog = captureLogger('warn')
      try {
        const overwritten = networkPolicyMarkerConditions(
          { kind: 'reconcile', summary: pass({ prune: 'pending' }) },
          [foreignMarker, foreignLegacy],
          now
        )

        expect(reasons(overwritten)).toEqual(['WorkflowNetworkPoliciesConverged/PrunePending'])
        expect(warnLog).not.toHaveBeenCalled()
        // Liveness witness: the same input through a carrying writer warns.
        networkPolicyMarkerConditions({ kind: 'terminal' }, [foreignMarker], now)
        expect(warnLog).toHaveBeenCalledTimes(1)
      } finally {
        warnLog.mockRestore()
      }
    })
  })
})

// R4-L5: each combination of facts has its own reason, and the facts are read
// back from the reason alone. A reworded message must not change what a
// published marker means. R4-L6: the prune fact has three values (Table A),
// so there are five combinations with a marker.
describe('networkPolicyMarkerFacts', () => {
  const now = '2026-09-23T12:00:00.000Z'
  const combinations = [
    { apply: 'pending', prune: 'converged' },
    { apply: 'converged', prune: 'pending' },
    { apply: 'converged', prune: 'unevaluated' },
    { apply: 'pending', prune: 'pending' },
    { apply: 'pending', prune: 'unevaluated' },
  ] as const

  it('R4-L5: a pending apply and a pending prune publish their own reason', () => {
    const marker = buildNetworkPolicyConvergedCondition({ apply: 'pending', prune: 'pending' }, now)
    expect(marker?.reason).toBe('RetryAndPrunePending')
  })

  it('R4-L6: an unevaluated prune publishes its own reason, alone or with a pending apply', () => {
    expect(
      buildNetworkPolicyConvergedCondition({ apply: 'converged', prune: 'unevaluated' }, now)
        ?.reason
    ).toBe('PruneUnevaluated')
    expect(
      buildNetworkPolicyConvergedCondition({ apply: 'pending', prune: 'unevaluated' }, now)?.reason
    ).toBe('RetryAndPruneUnevaluated')
  })

  it('R4-L6: converged facts publish no marker and read back from no marker', () => {
    expect(
      buildNetworkPolicyConvergedCondition({ apply: 'converged', prune: 'converged' }, now)
    ).toBeUndefined()
    expect(networkPolicyMarkerFacts([])).toStrictEqual({ apply: 'converged', prune: 'converged' })
  })

  it.each(combinations)(
    'R4-L5: reads %o back from the reason alone, whatever the message says',
    facts => {
      const marker = buildNetworkPolicyConvergedCondition(facts, now)
      // Liveness witness: the builder published a marker for these facts.
      expect(marker?.type).toBe('WorkflowNetworkPoliciesConverged')
      expect(marker?.status).toBe('False')
      expect(networkPolicyMarkerFacts([marker!])).toStrictEqual(facts)
      expect(networkPolicyMarkerFacts([{ ...marker!, message: 'reworded' }])).toStrictEqual(facts)
    }
  )

  it('R4-L5: every combination has a distinct reason', () => {
    const reasons = combinations.map(
      facts => buildNetworkPolicyConvergedCondition(facts, now)?.reason
    )
    expect(new Set(reasons).size).toBe(combinations.length)
  })

  it('reads the one historical RetryPending message that encoded a pending prune', () => {
    const historical = {
      type: 'WorkflowNetworkPoliciesConverged',
      status: 'False' as const,
      reason: 'RetryPending',
      message:
        'One or more run-lane NetworkPolicies are pending a retry (terminating or contended), and one or more the spec no longer wants are pending a delete',
      lastTransitionTime: now,
    }

    expect(networkPolicyMarkerFacts([historical])).toStrictEqual({
      apply: 'pending',
      prune: 'pending',
    })
    expect(
      networkPolicyMarkerFacts([{ ...historical, message: `${historical.message} ` }])
    ).toStrictEqual({
      apply: 'pending',
      prune: 'converged',
    })

    const settled = networkPolicyMarkerConditions(
      { kind: 'apply-retry', summary: { conflicts: [], retryPending: false } },
      [historical],
      now
    )
    expect(settled).toMatchObject([
      {
        type: 'WorkflowNetworkPoliciesConverged',
        status: 'False',
        reason: 'PrunePending',
      },
    ])
  })
})

describe('networkPolicyConditionsChanged', () => {
  const conflict = {
    type: 'WorkflowNetworkPolicyOwnership',
    status: 'False' as const,
    reason: 'OwnershipConflict',
    message: 'NetworkPolicy ownership conflict: test-wf-coord-to-wrc (owner-reference-mismatch)',
    lastTransitionTime: '2026-09-20T08:00:00.000Z',
  }
  const marker = {
    type: 'WorkflowNetworkPoliciesConverged',
    status: 'False' as const,
    reason: 'RetryPending',
    message: 'One or more run-lane NetworkPolicies are pending a retry (terminating or contended)',
    lastTransitionTime: '2026-09-20T08:00:00.000Z',
  }
  const unrelated = {
    type: 'Ready',
    status: 'True' as const,
    reason: 'Ready',
    message: 'ready',
    lastTransitionTime: '2026-09-20T08:00:00.000Z',
  }

  it('reports no change when the group matches, whatever lastTransitionTime or other types say', () => {
    expect(
      networkPolicyConditionsChanged(
        [conflict, marker, unrelated],
        [
          { ...conflict, lastTransitionTime: '2026-09-24T00:00:00.000Z' },
          { ...marker, lastTransitionTime: '2026-09-24T00:00:00.000Z' },
        ]
      )
    ).toBe(false)
  })

  it('reports a change when the retry marker appears', () => {
    expect(networkPolicyConditionsChanged([conflict], [conflict, marker])).toBe(true)
  })

  it('reports a change when the retry marker goes away', () => {
    expect(networkPolicyConditionsChanged([marker, unrelated], [])).toBe(true)
  })

  it('reports a change when the ownership condition message changes', () => {
    expect(networkPolicyConditionsChanged([conflict], [{ ...conflict, message: 'other' }])).toBe(
      true
    )
  })
})