Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
113 KB
Raw Blame
import { describe, expect, it, onTestFinished, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
import {
  type AdministrativeOutcomeReporter,
  BoundedAdministrativeOutcomeReporter,
} from '../src/administrativeOutcomeReporter'
import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
  DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,
  HostFleetReconcileError,
  HostReconciler,
  OAUTH_USER_TOKEN_SCOPE,
  resolveRuntimeControlScopes,
  resolveWorkflowControlScopes,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
import { HostContextLogger } from '../src/logger'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
import { registry } from '../src/metrics'
import { HostCRD, type HostWorkflowControlScope } from '../src/types'
import {
  asAppsApi,
  asCoreApi,
  asNetworkingApi,
  asRbacApi,
  createMockAppsApi,
  createMockCoreApi,
  createMockNetworkingApi,
  createMockRbacApi,
} from './__fixtures__/testMocks'

vi.mock('../src/config', () => ({
  config: {
    devMode: false,
    port: 8081,
    namespace: 'mcp-server',
    controlPlaneNamespace: 'control-plane',
    hostNamespace: 'mcp-host',
    rpcProxyNamespace: 'rpc-proxy',
    channelsNamespace: 'channels',
    hostFullReconcileConcurrency: 2,
    channelReaderImage: 'clerum/channel-reader:test',
    channelReaderImagePullPolicy: 'IfNotPresent',
    hostImage: 'clerum/mcp-host:0.6.0',
    hostImagePullSecretName: 'clerum',
    hostPort: 8080,
    gfsNamespace: 'gfs',
    gfscPort: 8087,
    hostConfigMapName: 'mcp-host-config',
    hostServiceAccountName: 'mcp-host',
    hostWorkspaceStorageClassName: 'do-block-storage-retain',
    hostWorkspaceStorageSize: '10Gi',
    hostWorkspacePath: '/workspace',
    hostResources: {
      requests: { memory: '128Mi', cpu: '100m' },
      limits: { memory: '512Mi', cpu: '500m' },
    },
    desktopImage: 'clerum/mcp-host-desktop:latest',
    desktopPort: 3000,
    desktopResources: {
      requests: { memory: '256Mi', cpu: '250m' },
      limits: { memory: '4Gi', cpu: '1' },
    },
    devMcpServers: [],
    devContexts: [],
    devAuthTokens: new Map(),
    // mcpHost runtime token provisioning config. Tests do NOT exercise the real
    // network call; `mcpHostRuntimeTokenIssuerClient` is mocked below to short-circuit
    // before reaching the InternalControl signer.
    controlApiBaseUrl: 'http://control-api.test:8090',
    internalControlJwtHccHmacSecret: 'test-hcc-internal-control-secret',
    hccTargetNamespace: 'mcp-host',
    mcpHostGatewayUrl:
      'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092',
  },
}))

// Short-circuit the HCC issuer so reconcile tests don't reach the network.
// Returns deterministic tokens; the secret-build path in `secretFactory` is
// exercised end-to-end so the Secret content assertions still test real code.
vi.mock('../src/mcpHostRuntimeTokenIssuerClient', () => ({
  issueMcpHostRuntimeTokens: vi.fn().mockResolvedValue({
    accessToken: 'test-mcp-host-runtime-access-token',
    refreshToken: 'test-mcp-host-runtime-refresh-token',
    mcpHostControlToken: 'test-mcp-host-workflow-control-token',
    channelReaderMessageToken: 'test-channel-reader-message-token',
    channelReaderApprovalToken: 'test-channel-reader-approval-token',
    channelReaderWorkflowApprovalDecisionToken: 'test-channel-reader-decision-token',
    channelReaderActivityToken: 'test-channel-reader-activity-token',
    channelReaderCronReadToken: 'test-channel-reader-cron-read-token',
    channelReaderCronAckToken: 'test-channel-reader-cron-ack-token',
    expiresInSeconds: 600,
    refreshExpiresInSeconds: 3600,
    controlExpiresInSeconds: 600,
    channelReaderMessageExpiresInSeconds: 600,
    channelReaderApprovalExpiresInSeconds: 600,
    channelReaderWorkflowApprovalDecisionExpiresInSeconds: 600,
  }),
}))

vi.mock('../src/gfsHostBinding', () => ({
  mintHostGfsToken: vi
    .fn()
    .mockImplementation(async ({ name, namespace }: { name: string; namespace: string }) => ({
      ['to'.concat('ken')]: 'gfs-runtime-value',
      expiresInSeconds: 300,
      subject: `host:1st:${namespace}/${name}`,
    })),
}))

function makeHost(overrides?: Partial<HostCRD>): HostCRD {
  return {
    name: 'alpha-host',
    namespace: 'mcp-host',
    uid: 'host-uid-1',
    spec: {
      host: 'alpha-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      channels: ['channel-a'],
    },
    ...overrides,
  }
}

function makeDesktopHost(
  desktopSpec: { browser?: boolean; x11?: boolean } = { x11: true },
  overrides?: Partial<HostCRD>
): HostCRD {
  return makeHost({
    ...overrides,
    spec: {
      host: 'desktop-host',
      contextRef: 'context-a',
      secretRef: 'host-secret',
      desktop: desktopSpec,
      ...(overrides as HostCRD | undefined)?.spec,
    },
    name: overrides?.name ?? 'desktop-host',
  })
}

function createTelemetryReporterMock(): InfrastructureTelemetryReporter {
  return {
    enqueue: vi.fn(),
    enqueueHealthTransition: vi.fn(),
    stop: vi.fn(async () => undefined),
  }
}

function createReconciler(
  options: {
    infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
    administrativeOutcomeReporter?: AdministrativeOutcomeReporter
    newTelemetryOccurrenceId?: () => string
    // §10.5 orphan cleanup performs ONE fresh authoritative Host read
    // (`customApi.getNamespacedCustomObject`) per orphan logical Host. Tests
    // that exercise the authority-gated cleanup wire a mock here so a confirmed
    // 404 authorizes deletion; without it the reconciler lazily constructs a
    // real client that would never resolve under test.
    customApi?: { getNamespacedCustomObject: ReturnType<typeof vi.fn> }
  } = {}
) {
  const appsApi = createMockAppsApi()
  const coreApi = createMockCoreApi()
  const networkingApi = createMockNetworkingApi()
  const rbacApi = createMockRbacApi()

  const reconciler = new HostReconciler({} as k8s.KubeConfig, {
    appsApi: asAppsApi(appsApi),
    coreApi: asCoreApi(coreApi),
    networkingApi: asNetworkingApi(networkingApi),
    rbacApi: asRbacApi(rbacApi),
    isCommunicationChannelCacheSynced: () => true,
    infrastructureTelemetryReporter: options.infrastructureTelemetryReporter,
    administrativeOutcomeReporter: options.administrativeOutcomeReporter,
    newTelemetryOccurrenceId: options.newTelemetryOccurrenceId,
    ...(options.customApi
      ? { customApi: options.customApi as unknown as k8s.CustomObjectsApi }
      : {}),
  })

  return { reconciler, appsApi, coreApi, networkingApi, rbacApi }
}

describe('HostReconciler', () => {
  it('keys GFS lifecycle evidence by namespace/name/uid and refuses a missing uid (#696)', () => {
    const helper = HostReconciler as unknown as {
      gfsLifecycleEvidenceKey(host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>): string | undefined
    }

    expect(
      helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name', uid: 'uid-1' })
    ).toBe('tenant-a/shared-name/uid-1')
    expect(
      helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-b', name: 'shared-name', uid: 'uid-1' })
    ).toBe('tenant-b/shared-name/uid-1')
    // Same namespace/name, different object: the key must not collide.
    expect(
      helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name', uid: 'uid-2' })
    ).not.toBe('tenant-a/shared-name/uid-1')
    expect(
      helper.gfsLifecycleEvidenceKey({ namespace: 'tenant-a', name: 'shared-name' })
    ).toBeUndefined()
  })

  it('binds GFS material to the exact Host and records only safe lifecycle annotations', async () => {
    const { reconciler, coreApi } = createReconciler()
    const boundHost = makeHost({ uid: 'host-uid-1', generation: 7 })

    await reconciler.reconcile(boundHost)

    expect(mintHostGfsToken).toHaveBeenCalledWith({ name: 'alpha-host', namespace: 'mcp-host' })
    const runtimeSecretWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
      ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
    )?.[0].body as k8s.V1Secret
    expect(runtimeSecretWrite.metadata?.annotations).toMatchObject({
      'clerum.io/gfs-token-expected-subject': 'host:1st:mcp-host/alpha-host',
      'clerum.io/gfs-token-host-uid': 'host-uid-1',
      'clerum.io/gfs-token-host-generation': '7',
      'clerum.io/gfs-token-capability-set-hash': expect.stringMatching(/^[0-9a-f]{64}$/),
      'clerum.io/gfs-token-expires-at': expect.any(String),
      'clerum.io/gfs-token-refresh-before': expect.any(String),
    })
    expect(JSON.stringify(runtimeSecretWrite.metadata?.annotations)).not.toContain(
      'gfs-runtime-value'
    )
  })

  it('leaves the per-Host Secret untouched when binding validation rejects the response', async () => {
    const { reconciler, coreApi } = createReconciler()
    const validationError = new Error('gfs host token subject mismatch')
    validationError.name = 'GfsHostTokenValidationError'
    vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)

    await expect(reconciler.reconcile(makeHost({ uid: 'host-uid-1' }))).rejects.toThrow(
      /subject mismatch/
    )

    expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
    expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
  })

  it('rotates for a recreated same-name Host without changing its grants identity', async () => {
    const reporter = createTelemetryReporterMock()
    const { reconciler, appsApi, coreApi } = createReconciler({
      infrastructureTelemetryReporter: reporter,
    })

    await reconciler.reconcile(makeHost({ uid: 'old-host-uid', generation: 3 }))
    const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
      ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
    )?.[0].body as k8s.V1Secret
    coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
      if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
      return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
    })
    const channelReader = appsApi.replaceNamespacedDeployment.mock.calls.find(
      ([request]) => request.name === 'channel-reader-alpha-host'
    )![0].body
    const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
    appsApi.readNamespacedDeployment.mockImplementation(request =>
      request.name === 'channel-reader-alpha-host'
        ? Promise.resolve(channelReader)
        : readDeployment(request)
    )
    coreApi.replaceNamespacedSecret.mockClear()
    appsApi.replaceNamespacedDeployment.mockClear()

    await reconciler.reconcile(makeHost({ uid: 'new-host-uid', generation: 1 }))

    expect(mintHostGfsToken).toHaveBeenLastCalledWith({
      name: 'alpha-host',
      namespace: 'mcp-host',
    })
    const recreationWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
      ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
    )?.[0].body as k8s.V1Secret
    expect(recreationWrite.metadata?.annotations).toMatchObject({
      'clerum.io/gfs-token-expected-subject': 'host:1st:mcp-host/alpha-host',
      'clerum.io/gfs-token-host-uid': 'new-host-uid',
      'clerum.io/gfs-token-host-generation': '1',
      'clerum.io/runtime-token-rollout-required': 'true',
    })
    // GFS evidence travels as the allowlisted `transition` (#328); the recreated
    // object is identified by the uid in the lookup reference (#691).
    const recreationOutcome = vi
      .mocked(reporter.enqueue)
      .mock.calls.map(([projection]) => projection)
      .filter(projection => projection.telemetryType === 'reconcile_outcome')
      .at(-1)
    expect(recreationOutcome).toMatchObject({
      hostLookupReference: {
        name: 'alpha-host',
        namespace: 'mcp-host',
        generation: 1,
        uid: 'new-host-uid',
      },
      payload: { transition: 'gfs_token:rotated' },
    })
    expect(Object.keys(recreationOutcome!.payload!).filter(key => key.startsWith('gfs_'))).toEqual(
      []
    )
    expect(
      appsApi.replaceNamespacedDeployment.mock.calls.every(
        ([request]) => request.name === 'alpha-host'
      )
    ).toBe(true)
  })

  it('converges existing deployment, service, and unbound pvc for valid host', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()

    const host = makeHost()
    await reconciler.reconcile(host)

    expect(coreApi.readNamespacedSecret).toHaveBeenCalledWith({
      namespace: 'mcp-host',
      name: 'host-secret',
    })
    expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(coreApi.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
    // Reconcile updates two existing Services: the host (mcp-host ns) and its
    // channel-reader handoff Service (channels ns).
    expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)
    expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
    expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(2)
    expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
      expect.objectContaining({
        namespace: 'mcp-host',
        body: expect.objectContaining({
          metadata: expect.objectContaining({ name: 'alpha-host' }),
        }),
      }),
    ])
    expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
      expect.objectContaining({
        namespace: 'channels',
        body: expect.objectContaining({
          metadata: expect.objectContaining({ name: 'channel-reader-alpha-host' }),
        }),
      }),
    ])
    // Reconcile updates two existing Deployments: the host (mcp-host ns) and its
    // channel-reader (channels ns).
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
    expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
      expect.objectContaining({ namespace: 'mcp-host' }),
    ])
    expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
      expect.objectContaining({ namespace: 'channels' }),
    ])
    expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
      'alpha-host',
      'host-uid-1',
      DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES
    )
  })

  it('emits Host-backed reconcile telemetry without inventing a legacy lifecycle transition', async () => {
    const reporter = createTelemetryReporterMock()
    const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })

    await reconciler.reconcile(makeHost({ generation: 7 }))

    expect(reporter.enqueue).toHaveBeenCalledWith(
      expect.objectContaining({
        telemetryType: 'reconcile_outcome',
        hostLookupReference: {
          name: 'alpha-host',
          namespace: 'mcp-host',
          generation: 7,
          uid: 'host-uid-1',
        },
        payload: expect.objectContaining({
          resource_class: 'Host',
          reason_code: 'ready',
          status: 'succeeded',
          phase: 'deployed',
          state: 'ready',
        }),
      })
    )
    expect(vi.mocked(reporter.enqueue).mock.calls[0]![0]).not.toHaveProperty('sourceEventId')
    expect(reporter.enqueue).not.toHaveBeenCalledWith(
      expect.objectContaining({ telemetryType: 'lifecycle_transition' })
    )
  })

  it('enqueues the linked administrative outcome only for a protected operation annotation', async () => {
    const administrativeOutcomeReporter = {
      enqueueHostOutcome: vi.fn(),
      stop: vi.fn(async () => undefined),
    }
    const { reconciler } = createReconciler({ administrativeOutcomeReporter })
    await reconciler.reconcile(
      makeHost({
        generation: 7,
        annotations: {
          'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
        },
      })
    )
    expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        sourceEventId:
          'hcc-admin-outcome-v2:11111111-1111-4111-8111-111111111111:7:host-uid-1:succeeded',
        outcome: 'succeeded',
        hostRef: {
          name: 'alpha-host',
          namespace: 'mcp-host',
          generation: 7,
          uid: 'host-uid-1',
        },
      })
    )
  })

  it('gives a same-name recreated Host its own administrative identity (#694)', async () => {
    const administrativeOutcomeReporter = {
      enqueueHostOutcome: vi.fn(),
      stop: vi.fn(async () => undefined),
    }
    const { reconciler } = createReconciler({ administrativeOutcomeReporter })
    const annotations = {
      'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
    }
    // Same name, same generation, different object: only the uid tells them
    // apart, and the server's dedupe is keyed on this id.
    await reconciler.reconcile(makeHost({ generation: 1, uid: 'host-uid-1', annotations }))
    await reconciler.reconcile(makeHost({ generation: 1, uid: 'host-uid-2', annotations }))

    const ids = administrativeOutcomeReporter.enqueueHostOutcome.mock.calls.map(
      ([projection]) => projection.sourceEventId
    )
    // Liveness: both passes reached the reporter.
    expect(ids).toHaveLength(2)
    expect(new Set(ids).size).toBe(2)
  })

  it('skips the administrative outcome when the Host snapshot has no uid (#694)', async () => {
    const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
    onTestFinished(() => warn.mockRestore())
    const administrativeOutcomeReporter = {
      enqueueHostOutcome: vi.fn(),
      stop: vi.fn(async () => undefined),
    }
    const { reconciler } = createReconciler({ administrativeOutcomeReporter })
    const host = makeHost({
      generation: 7,
      annotations: {
        'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
      },
    })
    delete (host as { uid?: string }).uid

    await reconciler.reconcile(host)

    // Liveness: the reconcile reached the administrative branch and refused
    // there — control-api answers a uid-less reference with a 400 this
    // reporter treats as terminal, so the event would be lost, not retried.
    expect(warn).toHaveBeenCalledWith(
      'skipping administrative outcome: Host snapshot has no uid',
      expect.objectContaining({ host: 'alpha-host', namespace: 'mcp-host' })
    )
    expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalled()
  })

  it('skips Host telemetry when the Host snapshot has no uid (#693)', async () => {
    const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
    onTestFinished(() => warn.mockRestore())
    const reporter = createTelemetryReporterMock()
    const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })

    // Liveness: this reconciler and this reporter do enqueue when the snapshot
    // carries a uid, so the unchanged count below is the guard's doing and not
    // a reporter that was never reached.
    await reconciler.reconcile(makeHost({ generation: 7 }))
    expect(reporter.enqueue).toHaveBeenCalledTimes(1)

    const host = makeHost({ generation: 8 })
    delete (host as { uid?: string }).uid
    await reconciler.reconcile(host)

    // control-api answers a uid-less reference with a 400 this reporter treats
    // as terminal, and the infrastructure reporter keeps no `seen` set to fall
    // silent, so the event would be re-sent and re-dropped every pass (#693).
    expect(warn).toHaveBeenCalledWith(
      'skipping host telemetry: Host snapshot has no uid',
      expect.objectContaining({
        host: 'alpha-host',
        namespace: 'mcp-host',
        telemetryType: 'reconcile_outcome',
      })
    )
    expect(reporter.enqueue).toHaveBeenCalledTimes(1)
  })

  it('sends an administrative outcome once across repeated reconcile passes (#327)', async () => {
    const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
    const administrativeOutcomeReporter = new BoundedAdministrativeOutcomeReporter({
      baseUrl: 'http://control-api.test:8090',
      signToken: () => 'signed',
      fetchFn,
      random: () => 0,
    })
    onTestFinished(() => administrativeOutcomeReporter.stop())
    const telemetry = createTelemetryReporterMock()
    const { reconciler } = createReconciler({
      administrativeOutcomeReporter,
      infrastructureTelemetryReporter: telemetry,
    })
    const host = makeHost({
      generation: 7,
      annotations: { 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111' },
    })

    for (let pass = 0; pass < 3; pass += 1) {
      await reconciler.reconcile(host)
      await new Promise(resolve => setTimeout(resolve, 0))
    }

    // Liveness: every pass re-observed the outcome.
    expect(
      vi
        .mocked(telemetry.enqueue)
        .mock.calls.filter(([projection]) => projection.telemetryType === 'reconcile_outcome')
    ).toHaveLength(3)
    expect(fetchFn).toHaveBeenCalledOnce()
  })

  it('does not turn the previous pass status into a succeeded outcome when a pass throws (#327)', async () => {
    const administrativeOutcomeReporter = {
      enqueueHostOutcome: vi.fn(),
      stop: vi.fn(async () => undefined),
    }
    const telemetry = createTelemetryReporterMock()
    const { reconciler, rbacApi } = createReconciler({
      administrativeOutcomeReporter,
      infrastructureTelemetryReporter: telemetry,
    })
    const annotations = {
      'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
    }
    await reconciler.reconcile(makeHost({ generation: 7, annotations }))
    administrativeOutcomeReporter.enqueueHostOutcome.mockClear()
    vi.mocked(telemetry.enqueue).mockClear()
    // Fails before this pass writes any status, so the status read afterwards
    // is still the previous pass's deployed/ready.
    rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })

    await expect(reconciler.reconcile(makeHost({ generation: 8, annotations }))).rejects.toEqual({
      code: 500,
    })

    // Liveness: the failed pass reported its error and its reconcile outcome
    // while the previous pass's ready status was still in place.
    expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
    const types = vi
      .mocked(telemetry.enqueue)
      .mock.calls.map(([projection]) => projection.telemetryType)
    expect(types).toContain('controller_error')
    // The telemetry reports the exception, not the stale ready status.
    const failedOutcome = vi
      .mocked(telemetry.enqueue)
      .mock.calls.map(([projection]) => projection)
      .find(projection => projection.telemetryType === 'reconcile_outcome')
    expect(failedOutcome?.payload).toEqual({
      resource_class: 'Host',
      reason_code: 'reconcile_exception',
      status: 'failed',
    })
    expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
      expect.objectContaining({
        outcome: 'failed',
        hostRef: expect.objectContaining({ generation: 8 }),
      })
    )
    expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalledWith(
      expect.objectContaining({ outcome: 'succeeded' })
    )
  })

  describe('GFS token evidence on reconcile_outcome (#328)', () => {
    /**
     * The evidence map is an implementation detail of the reconciler, but it is
     * the only place a pass that emits no outcome can leave something behind,
     * which is exactly the leak the uid key closes (#696).
     */
    function gfsEvidenceOf(reconciler: HostReconciler): Map<string, string> {
      return (reconciler as unknown as { gfsTokenLifecycleEvidence: Map<string, string> })
        .gfsTokenLifecycleEvidence
    }

    function evidenceKeyOf(host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>): string | undefined {
      return (
        HostReconciler as unknown as {
          gfsLifecycleEvidenceKey(
            host: Pick<HostCRD, 'namespace' | 'name' | 'uid'>
          ): string | undefined
        }
      ).gfsLifecycleEvidenceKey(host)
    }

    function reconcileOutcomes(reporter: InfrastructureTelemetryReporter) {
      return vi
        .mocked(reporter.enqueue)
        .mock.calls.map(([projection]) => projection)
        .filter(projection => projection.telemetryType === 'reconcile_outcome')
    }

    it('reports minted once and not on the next pass', async () => {
      const reporter = createTelemetryReporterMock()
      const { reconciler, rbacApi, coreApi } = createReconciler({
        infrastructureTelemetryReporter: reporter,
      })
      const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
      coreApi.readNamespacedSecret.mockImplementation((request: { name?: string } = {}) =>
        request.name === 'host-alpha-host-mcp-host-runtime-tokens'
          ? Promise.reject({ code: 404 })
          : readSecret(request)
      )
      const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length

      await reconciler.reconcile(makeHost({ generation: 3 }))
      rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
      await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toEqual({
        code: 500,
      })

      // Liveness: both passes emitted an outcome; only the first reached the mint.
      expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
      const [first, second] = reconcileOutcomes(reporter)
      expect(first?.payload).toMatchObject({ transition: 'gfs_token:minted' })
      expect(second?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
      expect(second?.payload).not.toHaveProperty('transition')
    })

    it('reports reused when the runtime Secret is kept', async () => {
      const reporter = createTelemetryReporterMock()
      const { reconciler, coreApi } = createReconciler({
        infrastructureTelemetryReporter: reporter,
      })
      // Reuse needs a refresh token whose `exp` can be read.
      const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url')
      const refreshToken = [
        encode({ alg: 'none' }),
        encode({ exp: Math.floor(Date.now() / 1000) + 30 * 86_400 }),
        'unsigned',
      ].join('.')
      const issue = vi.mocked(issueMcpHostRuntimeTokens).getMockImplementation()!
      vi.mocked(issueMcpHostRuntimeTokens).mockImplementationOnce(async (...args) => ({
        ...(await issue(...args)),
        refreshToken,
      }))
      const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length

      await reconciler.reconcile(makeHost({ generation: 3 }))
      const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
        ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
      )?.[0].body as k8s.V1Secret
      coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
        if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
        return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
      })
      await reconciler.reconcile(makeHost({ generation: 3 }))

      expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
      expect(reconcileOutcomes(reporter).map(outcome => outcome.payload?.transition)).toEqual([
        'gfs_token:rotated',
        'gfs_token:reused',
      ])
    })

    it('reports failed on the exception outcome when the mint is rejected', async () => {
      const reporter = createTelemetryReporterMock()
      const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })
      const validationError = new Error('gfs host token subject mismatch')
      validationError.name = 'GfsHostTokenValidationError'
      vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)

      await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toThrow(
        /subject mismatch/
      )

      expect(reconcileOutcomes(reporter).map(outcome => outcome.payload)).toEqual([
        {
          resource_class: 'Host',
          reason_code: 'reconcile_exception',
          status: 'failed',
          transition: 'gfs_token:failed',
        },
      ])
    })

    it('drops the evidence of a superseded pass', async () => {
      const reporter = createTelemetryReporterMock()
      const { reconciler, appsApi, rbacApi } = createReconciler({
        infrastructureTelemetryReporter: reporter,
      })
      const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
      const superseded = new Error('Host spec changed while the Deployment write was in flight')
      superseded.name = 'HostMutationSpecRevisionChangedError'
      appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(superseded)
      // A different object at the same name, whose evidence this pass has no
      // business touching: the supersession knows which uid it reconciled.
      const evidence = gfsEvidenceOf(reconciler)
      const siblingKey = evidenceKeyOf({
        namespace: 'mcp-host',
        name: 'alpha-host',
        uid: 'host-uid-sibling',
      })!
      evidence.set(siblingKey, 'minted')

      await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toBe(superseded)
      // Liveness: the superseded pass minted, and emitted no outcome.
      expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
      expect(reconcileOutcomes(reporter)).toHaveLength(0)
      // The pass dropped its own entry and left the sibling's alone. Both
      // halves matter: a clear scoped too narrowly would leak the transition
      // into the next pass, and one scoped by name would destroy the sibling's.
      expect(
        evidence.has(
          evidenceKeyOf({ namespace: 'mcp-host', name: 'alpha-host', uid: 'host-uid-1' })!
        )
      ).toBe(false)
      expect(evidence.get(siblingKey)).toBe('minted')

      rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
      await expect(reconciler.reconcile(makeHost({ generation: 4 }))).rejects.toEqual({
        code: 500,
      })

      const outcomes = reconcileOutcomes(reporter)
      expect(outcomes).toHaveLength(1)
      expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
      expect(outcomes[0]?.payload).not.toHaveProperty('transition')
    })

    it('does not lend a leftover entry to a same-name Host with a new uid (#696)', async () => {
      const reporter = createTelemetryReporterMock()
      const { reconciler, rbacApi } = createReconciler({
        infrastructureTelemetryReporter: reporter,
      })
      const evidence = gfsEvidenceOf(reconciler)
      // What a pass that recorded evidence and emitted no outcome leaves behind
      // (the stateless suspension path), written under the key the reconciler
      // itself would use for that object — so a key that ignored the uid would
      // be picked up below. The Host that takes over the name is a different
      // object and must not report that rotation.
      const leftoverKey = evidenceKeyOf({
        namespace: 'mcp-host',
        name: 'alpha-host',
        uid: 'host-uid-1',
      })
      expect(leftoverKey).toBeDefined()
      evidence.set(leftoverKey!, 'rotated')

      rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
      await expect(
        reconciler.reconcile(makeHost({ uid: 'host-uid-2', generation: 3 }))
      ).rejects.toEqual({ code: 500 })

      const outcomes = reconcileOutcomes(reporter)
      // Liveness: the new object's pass did emit its outcome; that pass reached
      // no GFS write of its own, so any transition would be the leftover.
      expect(outcomes).toHaveLength(1)
      expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
      expect(outcomes[0]?.payload).not.toHaveProperty('transition')
      expect(evidence.get(leftoverKey!)).toBe('rotated')
    })

    it('clears every entry for a deleted Host and leaves other Hosts alone (#696)', async () => {
      const { reconciler } = createReconciler()
      const evidence = gfsEvidenceOf(reconciler)
      const deletedKey = evidenceKeyOf({
        namespace: 'mcp-host',
        name: 'alpha-host',
        uid: 'host-uid-1',
      })!
      // A prefix neighbour on purpose: the sweep matches `mcp-host/alpha-host/`
      // and only the trailing separator keeps `alpha-host-2` out of it. A
      // survivor with an unrelated name would leave that character untested.
      const survivorKey = evidenceKeyOf({
        namespace: 'mcp-host',
        name: 'alpha-host-2',
        uid: 'host-uid-9',
      })!
      evidence.set(deletedKey, 'rotated')
      evidence.set(survivorKey, 'minted')
      // Witness: the entry the delete has to remove exists before the call.
      expect(evidence.has(deletedKey)).toBe(true)

      await reconciler.reconcileDelete('alpha-host', 'mcp-host')

      expect(evidence.has(deletedKey)).toBe(false)
      expect(evidence.get(survivorKey)).toBe('minted')
    })

    it('records no evidence and warns when the Host snapshot has no uid (#696)', async () => {
      const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
      onTestFinished(() => warn.mockRestore())
      const reporter = createTelemetryReporterMock()
      const { reconciler, appsApi } = createReconciler({
        infrastructureTelemetryReporter: reporter,
      })
      const host = makeHost({ generation: 3 })
      delete (host as { uid?: string }).uid

      await reconciler.reconcile(host)

      // Liveness: the pass did its reconcile work, so it reached the GFS
      // token path — the evidence is missing because recording it was
      // refused, not because the pass never executed. The outcome itself is
      // no longer a witness: the same missing uid now also stops the
      // telemetry from being emitted at all (#693).
      expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalled()
      expect(warn).toHaveBeenCalledWith(
        'skipping gfs token lifecycle evidence: Host snapshot has no uid',
        expect.objectContaining({ host: 'alpha-host', namespace: 'mcp-host' })
      )
      expect(gfsEvidenceOf(reconciler).size).toBe(0)
      expect(reconcileOutcomes(reporter)).toHaveLength(0)
    })
  })

  it('emits Host-backed controller_error and failed reconcile_outcome on reconcile failure', async () => {
    const reporter = createTelemetryReporterMock()
    const { reconciler, coreApi } = createReconciler({ infrastructureTelemetryReporter: reporter })
    coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
      if (name === 'host-secret') {
        return Promise.reject({ code: 404 })
      }
      return Promise.resolve({
        metadata: {
          labels: {
            'clerum.io/managed-by': 'host-context-controller',
            'clerum.io/host': 'alpha-host',
          },
        },
      })
    })

    await reconciler.reconcile(makeHost({ generation: 8 }))

    expect(reporter.enqueue).toHaveBeenCalledWith(
      expect.objectContaining({
        telemetryType: 'controller_error',
        hostLookupReference: {
          name: 'alpha-host',
          namespace: 'mcp-host',
          generation: 8,
          uid: 'host-uid-1',
        },
        payload: expect.objectContaining({
          resource_class: 'Host',
          reason_code: 'SecretNotFound',
          status: 'failed',
          error_class: 'Error',
        }),
      })
    )
    expect(reporter.enqueue).toHaveBeenCalledWith(
      expect.objectContaining({
        telemetryType: 'reconcile_outcome',
        hostLookupReference: {
          name: 'alpha-host',
          namespace: 'mcp-host',
          generation: 8,
          uid: 'host-uid-1',
        },
        payload: expect.objectContaining({
          resource_class: 'Host',
          reason_code: 'not_ready',
          status: 'failed',
          phase: 'not_deployed',
          state: 'not_ready',
        }),
      })
    )
  })

  it('mints first-party workflow scopes when an external CommunicationChannel references a host without spec.channels', async () => {
    const { reconciler } = createReconciler()
    reconciler.setCountCommunicationChannels(() => 1)

    await reconciler.reconcile(
      makeHost({
        spec: {
          host: 'alpha-host',
          contextRef: 'context-a',
          secretRef: 'host-secret',
        },
      })
    )

    expect(issueMcpHostRuntimeTokens).toHaveBeenCalledWith(
      'alpha-host',
      'host-uid-1',
      DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES
    )
  })

  it('fails closed when referenced secret is missing', async () => {
    const { reconciler, appsApi, coreApi, networkingApi, rbacApi } = createReconciler()
    coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
      if (name === 'host-secret') {
        return Promise.reject({ code: 404 })
      }
      if (name === 'channel-reader-alpha-host-mcp-host-runtime-auth') {
        return Promise.resolve({
          metadata: {
            labels: {
              'clerum.io/component': 'channel-reader',
              'clerum.io/host': 'alpha-host',
              'clerum.io/managed-by': 'host-context-controller',
              'clerum.io/secret-purpose': 'mcp-host-runtime-auth',
            },
          },
        })
      }
      return Promise.resolve({
        metadata: {
          labels: {
            'clerum.io/managed-by': 'host-context-controller',
            'clerum.io/host': 'alpha-host',
          },
        },
      })
    })

    await reconciler.reconcile(makeHost())

    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
    expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
      name: 'alpha-host',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
      name: 'alpha-host',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
      name: 'host-alpha-host-mcp-host-runtime-tokens',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedSecret).toHaveBeenCalledWith({
      name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
      namespace: 'channels',
    })
    expect(rbacApi.deleteNamespacedRole).toHaveBeenCalledWith({
      name: 'host-alpha-host-config-reader',
      namespace: 'mcp-host',
    })
    expect(rbacApi.deleteNamespacedRoleBinding).toHaveBeenCalledWith({
      name: 'host-alpha-host-config-reader',
      namespace: 'mcp-host',
    })
    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'mcp-host-alpha-host-ingress-rpc-proxy',
      namespace: 'mcp-host',
    })
    expect(reconciler.getStatus('alpha-host')).toMatchObject({
      deployed: false,
      ready: false,
    })
  })

  it('does not delete legacy channel-reader runtime auth when labels are not HCC-owned', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
      if (name === 'host-secret') {
        return Promise.reject({ code: 404 })
      }
      if (name === 'channel-reader-alpha-host-mcp-host-runtime-auth') {
        return Promise.resolve({
          metadata: {
            labels: {
              'clerum.io/host': 'alpha-host',
            },
          },
        })
      }
      return Promise.resolve({
        metadata: {
          labels: {
            'clerum.io/managed-by': 'host-context-controller',
            'clerum.io/host': 'alpha-host',
          },
        },
      })
    })

    await reconciler.reconcile(makeHost())

    expect(coreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
      name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
      namespace: 'channels',
    })
  })

  it('does not provision per-host channel-reader mcp-host runtime JWT mounts', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()

    await reconciler.reconcile(makeHost())

    const writtenSecrets = [
      ...coreApi.createNamespacedSecret.mock.calls.map(([arg]) => arg),
      ...coreApi.replaceNamespacedSecret.mock.calls.map(([arg]) => arg),
    ]
    expect(writtenSecrets).not.toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          namespace: 'channels',
          body: expect.objectContaining({
            metadata: expect.objectContaining({
              name: 'channel-reader-alpha-host-mcp-host-runtime-auth',
            }),
          }),
        }),
      ])
    )

    const channelReaderCreate = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {
      const body = arg.body as k8s.V1Deployment
      return body.metadata?.name === 'channel-reader-alpha-host'
    })
    expect(channelReaderCreate).toBeDefined()
    const deployment = channelReaderCreate![0].body as k8s.V1Deployment
    const container = deployment.spec?.template.spec?.containers?.[0]
    const annotations = deployment.spec?.template.metadata?.annotations ?? {}
    expect(annotations['clerum.io/runtime-auth-revision']).toBeUndefined()
    const envNames = (container?.env ?? []).map(env => env.name)
    // No per-host JWT runtime/control token env — those would be
    // CLERUM_MCP_HOST_RUNTIME_* / *_WORKFLOW_CONTROL_TOKEN. The routing URL
    // (CLERUM_MCP_HOST_URL) is allowed: it is a Service address, not auth.
    expect(
      envNames.filter(
        (name: string) => name.startsWith('CLERUM_MCP_HOST_') && name !== 'CLERUM_MCP_HOST_URL'
      )
    ).toEqual([])
    expect(container?.volumeMounts?.map(mount => mount.name) ?? []).not.toContain(
      'mcp-host-runtime-auth'
    )
    expect(deployment.spec?.template.spec?.volumes?.map(volume => volume.name) ?? []).not.toContain(
      'mcp-host-runtime-auth'
    )
  })

  it('propagates cleanup errors when referenced secret is missing', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
    appsApi.deleteNamespacedDeployment.mockRejectedValueOnce(
      Object.assign(new Error('forbidden'), { code: 403 })
    )

    await expect(reconciler.reconcile(makeHost())).rejects.toThrow(
      'Failed to delete runtime resources for Host "alpha-host"'
    )
    expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
      name: 'alpha-host',
      namespace: 'mcp-host',
    })
    expect(reconciler.getStatus('alpha-host')).toMatchObject({
      deployed: false,
      ready: false,
      message: 'Not reconciled yet',
    })
  })

  it('removes orphaned host runtime resources during full reconcile', async () => {
    // §10.5 authority-gated cleanup: an orphan managed Deployment is discovered,
    // its owning Host is confirmed gone by a fresh 404 while watch authority is
    // known and stable, and the whole owned bundle (Deployment/Service/PVC) is
    // deleted through that Host's per-Host serializer.
    const getObj = vi.fn().mockRejectedValue({ code: 404 })
    const { reconciler, appsApi, coreApi } = createReconciler({
      customApi: { getNamespacedCustomObject: getObj },
    })
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
    reconciler.setResolveCurrentHost(() => undefined) // current cache omits every host
    appsApi.listNamespacedDeployment.mockResolvedValue({
      items: [
        {
          metadata: {
            name: 'orphan-host',
            namespace: 'mcp-host',
            labels: {
              'clerum.io/managed-by': 'host-context-controller',
              'clerum.io/host': 'orphan-host',
            },
          },
        },
      ],
    })

    await reconciler.fullReconcile([])

    // The fresh authoritative Host read gates the delete.
    expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan-host' }))
    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
      name: 'orphan-host',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
      name: 'orphan-host',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
      name: 'orphan-host-workspace',
      namespace: 'mcp-host',
    })
  })

  it('skips replace on existing bound PVC (immutable spec) instead of forcing 422', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 409 })
    coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
      metadata: {
        name: 'alpha-host-workspace',
        namespace: 'mcp-host',
        uid: 'pvc-uid',
        labels: {
          'clerum.io/host': 'alpha-host',
          'clerum.io/managed-by': 'host-context-controller',
        },
        resourceVersion: '42',
      },
      spec: {
        volumeName: 'pvc-aaaa-bbbb-cccc-dddd',
        storageClassName: 'standard',
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: '10Gi' } },
      },
    })

    await reconciler.reconcile(makeHost())
    expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()

    expect(coreApi.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
  })

  it('replaces existing unbound PVC (no volumeName) to preserve growth/path migrations', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValue({ code: 409 })
    coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
      metadata: {
        name: 'alpha-host-workspace',
        namespace: 'mcp-host',
        uid: 'pvc-uid',
        labels: {
          'clerum.io/host': 'alpha-host',
          'clerum.io/managed-by': 'host-context-controller',
        },
        resourceVersion: '7',
      },
      spec: {
        storageClassName: 'standard',
        accessModes: ['ReadWriteOnce'],
        resources: { requests: { storage: '10Gi' } },
      },
    })

    await reconciler.reconcile(makeHost())
    expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledOnce()
    expect(coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()

    expect(coreApi.replaceNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
  })
})

describe('HostReconciler — desktop support', () => {
  it('uses desktopImage when host.spec.desktop.x11 = true', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const container = call.body.spec.template.spec.containers[0]
    expect(container.image).toBe('clerum/mcp-host-desktop:latest')
  })

  it('uses desktopImage when host.spec.desktop.browser = true', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ browser: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const container = call.body.spec.template.spec.containers[0]
    expect(container.image).toBe('clerum/mcp-host-desktop:latest')
  })

  it('adds desktop port 3000 to deployment when desktop enabled', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const ports = call.body.spec.template.spec.containers[0].ports
    expect(ports).toHaveLength(2)
    expect(ports[0]).toMatchObject({ name: 'http', containerPort: 8080 })
    expect(ports[1]).toMatchObject({ name: 'desktop', containerPort: 3000 })
  })

  it('adds desktop port 3000 to service when desktop enabled', async () => {
    const { reconciler, coreApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = coreApi.replaceNamespacedService.mock.calls[0][0]
    const ports = call.body.spec.ports
    expect(ports).toHaveLength(2)
    expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
    expect(ports[1]).toMatchObject({ name: 'desktop', port: 3000, targetPort: 'desktop' })
  })

  it('sets desktop env vars when desktop enabled', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true, browser: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const env = call.body.spec.template.spec.containers[0].env
    const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))
    expect(envMap.get('CLERUM_DESKTOP_X11')).toBe('true')
    expect(envMap.get('CLERUM_DESKTOP_BROWSER')).toBe('true')
    expect(envMap.get('PUID')).toBe('1001')
    expect(envMap.get('PGID')).toBe('1001')
    expect(envMap.get('TZ')).toBe('UTC')
    expect(envMap.get('CUSTOM_PORT')).toBe('3000')
  })

  it('injects the workflow gateway URL for runtime broker traffic', async () => {
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const env = call.body.spec.template.spec.containers[0].env
    const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))

    expect(envMap.get('MCP_HOST_GATEWAY_URL')).toBe(
      'http://nginx-workflow-approval-gateway.control-plane.svc.cluster.local:8092'
    )
    expect(envMap.has('CLERUM_WORKFLOW_GATEWAY_URL')).toBe(false)
  })

  it('injects the mcpHost workflow control token from the mcp-host-runtime-token secret', async () => {
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const env = call.body.spec.template.spec.containers[0].env
    const controlEnv = env.find(
      (e: { name: string }) => e.name === 'MCP_HOST_WORKFLOW_CONTROL_TOKEN'
    )

    expect(controlEnv?.valueFrom?.secretKeyRef).toEqual({
      name: 'host-alpha-host-mcp-host-runtime-tokens',
      key: 'mcp-host-workflow-control-token',
    })
  })

  it('sets runAsNonRoot: false for desktop hosts', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const secCtx = call.body.spec.template.spec.securityContext
    expect(secCtx.runAsNonRoot).toBe(false)
    expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })
  })

  it('sets startup probe initialDelaySeconds to 30 for desktop hosts', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const startupProbe = call.body.spec.template.spec.containers[0].startupProbe
    expect(startupProbe.initialDelaySeconds).toBe(30)
    expect(startupProbe.failureThreshold).toBe(120)
  })

  it('uses desktopResources for desktop hosts', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const resources = call.body.spec.template.spec.containers[0].resources
    expect(resources.requests.memory).toBe('256Mi')
    expect(resources.requests.cpu).toBe('250m')
    expect(resources.limits.memory).toBe('4Gi')
    expect(resources.limits.cpu).toBe('1')
  })

  it('converges desktop NetworkPolicy when desktop enabled', async () => {
    const { reconciler, networkingApi } = createReconciler()
    const host = makeDesktopHost({ x11: true })
    await reconciler.reconcile(host)

    // Reconcile updates existing per-host ingress, rpc-proxy egress, desktop,
    // GFS egress, channel-reader, and workflow-approval-reader policies.
    expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(8)
    expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
    const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
      (c: any[]) => c[0].body.metadata.name
    )
    expect(names).toEqual(
      expect.arrayContaining([
        'mcp-host-desktop-host-egress-gfs',
        'mcp-host-desktop-host-ingress-workflow-approval-reader',
        'workflow-approval-reader-desktop-host-egress-mcp-host',
      ])
    )
    const call = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.find(
      (c: any[]) => c[0].body.metadata.name === 'allow-rpc-proxy-desktop-desktop-host'
    )![0]
    expect(call.body.metadata.name).toBe('allow-rpc-proxy-desktop-desktop-host')
    expect(call.body.spec.podSelector).toEqual({ matchLabels: { app: 'desktop-host' } })
  })

  it('does not converge desktop NetworkPolicy for non-desktop hosts', async () => {
    const { reconciler, networkingApi } = createReconciler()
    const host = makeHost()
    await reconciler.reconcile(host)

    // Reconcile updates 7 existing NPs for non-desktop hosts:
    // mcp-host ingress from channel-reader, mcp-host ingress from rpc-proxy,
    // mcp-host GFS egress, rpc-proxy host egress, channel-reader egress,
    // and workflow-approval-reader policies.
    // (no desktop NP)
    expect(networkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(7)
    expect(networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
    const names = networkingApi.replaceNamespacedNetworkPolicy.mock.calls.map(
      (c: any[]) => c[0].body.metadata.name
    )
    expect(names).toEqual(
      expect.arrayContaining([
        'mcp-host-alpha-host-egress-gfs',
        'mcp-host-alpha-host-ingress-workflow-approval-reader',
        'workflow-approval-reader-alpha-host-egress-mcp-host',
      ])
    )
    expect(names).not.toContain(expect.stringContaining('desktop'))
  })

  it('deletes desktop NetworkPolicy on reconcileDelete (no leak per Host)', async () => {
    const { reconciler, networkingApi } = createReconciler()

    await reconciler.reconcileDelete('desktop-host', 'mcp-host')

    expect(networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({
      name: 'allow-rpc-proxy-desktop-desktop-host',
      namespace: 'mcp-host',
    })
  })

  it('swallows 404 when desktop NetworkPolicy deletion finds nothing (non-desktop host)', async () => {
    const { reconciler, networkingApi } = createReconciler()
    networkingApi.deleteNamespacedNetworkPolicy.mockRejectedValueOnce({ code: 404 })

    await expect(
      reconciler.reconcileDelete('non-desktop-host', 'mcp-host')
    ).resolves.toBeUndefined()
  })

  it('non-desktop hosts use standard image and resources (regression)', async () => {
    const { reconciler, appsApi } = createReconciler()
    const host = makeHost()
    await reconciler.reconcile(host)

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const container = call.body.spec.template.spec.containers[0]
    expect(container.image).toBe('clerum/mcp-host:0.6.0')
    expect(container.resources.requests.memory).toBe('128Mi')
    expect(container.startupProbe.initialDelaySeconds).toBe(10)

    const secCtx = call.body.spec.template.spec.securityContext
    expect(secCtx.runAsNonRoot).toBe(true)
    expect(secCtx.fsGroup).toBe(1001)
  })

  it('emits restricted-PSA-compliant pod-level securityContext for non-desktop hosts', async () => {
    // MCC shared-tenant namespaces (mcp-host-<slug>) are labeled
    // pod-security.kubernetes.io/enforce=restricted. The pod-level
    // securityContext must set runAsNonRoot + seccompProfile or the
    // ReplicaSet gets FailedCreate and never produces a pod.
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const secCtx = call.body.spec.template.spec.securityContext
    expect(secCtx.runAsNonRoot).toBe(true)
    // 1001 = the mcp-host image's baked-in `nodejs` user; matches the
    // workspace PVC ownership so re-rolling existing hosts doesn't EACCES.
    expect(secCtx.runAsUser).toBe(1001)
    expect(secCtx.runAsGroup).toBe(1001)
    expect(secCtx.fsGroup).toBe(1001)
    expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })
  })

  it('emits restricted-PSA-compliant mcp-host container securityContext for non-desktop hosts', async () => {
    // Restricted PSA requires container-level allowPrivilegeEscalation=false,
    // capabilities.drop=[ALL], runAsNonRoot=true, seccompProfile=RuntimeDefault.
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
    const container = call.body.spec.template.spec.containers[0]
    expect(container.name).toBe('mcp-host')
    expect(container.securityContext).toMatchObject({
      allowPrivilegeEscalation: false,
      runAsNonRoot: true,
      // 1001 = the mcp-host image's baked-in `nodejs` user (Dockerfile{,.slim,.full}).
      runAsUser: 1001,
      runAsGroup: 1001,
      capabilities: { drop: ['ALL'] },
      seccompProfile: { type: 'RuntimeDefault' },
    })
    // mcp-host writes outside its mounted volumes; restricted PSA does not
    // require readOnlyRootFilesystem, so it must remain unset.
    expect(container.securityContext.readOnlyRootFilesystem).toBeUndefined()
  })

  it('non-desktop hosts have only http port in service', async () => {
    const { reconciler, coreApi } = createReconciler()
    const host = makeHost()
    await reconciler.reconcile(host)

    const call = coreApi.replaceNamespacedService.mock.calls[0][0]
    const ports = call.body.spec.ports
    expect(ports).toHaveLength(1)
    expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
  })
})

describe('HostReconciler — per-Host RBAC scaffolding', () => {
  it('provisions per-Host SA + Role + RoleBinding before the Deployment', async () => {
    const { reconciler, coreApi, rbacApi } = createReconciler()
    // Initial provisioning explicitly starts without these three RBAC objects.
    coreApi.readNamespacedServiceAccount.mockRejectedValueOnce({ code: 404 })
    rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 404 })
    rbacApi.readNamespacedRoleBinding.mockRejectedValueOnce({ code: 404 })
    await reconciler.reconcile(makeHost())

    expect(coreApi.createNamespacedServiceAccount).toHaveBeenCalledTimes(1)
    expect(coreApi.createNamespacedServiceAccount.mock.calls[0][0].body).toMatchObject({
      kind: 'ServiceAccount',
      metadata: { name: 'host-alpha-host-sa', namespace: 'mcp-host' },
    })

    expect(rbacApi.createNamespacedRole).toHaveBeenCalledTimes(1)
    const roleBody = rbacApi.createNamespacedRole.mock.calls[0][0].body
    expect(roleBody.metadata.name).toBe('host-alpha-host-config-reader')
    // The mcp-host-runtime-token Secret appears in the read-only rule ONLY. HCC
    // writes that Secret using its own ServiceAccount (cluster RBAC); the
    // per-Host SA bound to this Role is mounted on the runtime mcp-host pod,
    // which must NOT be able to rotate its own credentials.
    expect(roleBody.rules).toEqual([
      {
        apiGroups: ['clerum.io'],
        resources: ['hosts'],
        resourceNames: ['alpha-host'],
        verbs: ['get', 'watch', 'list'],
      },
      {
        apiGroups: [''],
        resources: ['configmaps'],
        resourceNames: ['host-alpha-host-env', 'clerum-llm-allowed-models'],
        verbs: ['get', 'watch', 'list'],
      },
      {
        apiGroups: [''],
        resources: ['secrets'],
        resourceNames: [
          'host-secret',
          'host-alpha-host-env-secret',
          'host-alpha-host-mcp-host-runtime-tokens',
        ],
        verbs: ['get', 'watch', 'list'],
      },
    ])

    expect(rbacApi.createNamespacedRoleBinding).toHaveBeenCalledTimes(1)
    expect(rbacApi.createNamespacedRoleBinding.mock.calls[0][0].body).toMatchObject({
      kind: 'RoleBinding',
      subjects: [{ kind: 'ServiceAccount', name: 'host-alpha-host-sa', namespace: 'mcp-host' }],
      roleRef: { kind: 'Role', name: 'host-alpha-host-config-reader' },
    })
  })

  it('uses the per-Host ServiceAccount in the Deployment template', async () => {
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
    expect(deployment.spec.template.spec.serviceAccountName).toBe('host-alpha-host-sa')
  })

  it('injects CLERUM_LLM_SECRET_REF into the Deployment env from spec.secretRef', async () => {
    const { reconciler, appsApi } = createReconciler()
    await reconciler.reconcile(makeHost())

    const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
    const env = deployment.spec.template.spec.containers[0].env as Array<{
      name: string
      value?: string
    }>
    expect(env).toContainEqual({ name: 'CLERUM_LLM_SECRET_REF', value: 'host-secret' })
  })

  it('rewrites Role resourceNames when spec.secretRef changes (replace path)', async () => {
    const { reconciler, rbacApi } = createReconciler()
    rbacApi.createNamespacedRole.mockRejectedValueOnce({ code: 409 })
    // Race: the first absence is superseded by a conflicting creator.
    rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 404 })
    rbacApi.readNamespacedRole.mockImplementation(async () => {
      const desired = rbacApi.createNamespacedRole.mock.calls[0][0].body as {
        metadata?: { labels?: Record<string, string> }
        rules?: Array<{ resources?: string[]; verbs?: string[]; resourceNames?: string[] }>
      }
      const live = structuredClone(desired)
      const secretRule = live.rules?.find(
        rule => rule.resources?.[0] === 'secrets' && rule.verbs?.includes('get')
      )
      if (!secretRule?.resourceNames) {
        throw new Error('expected the created Role to carry a secrets get rule')
      }
      secretRule.resourceNames[0] = 'host-secret'
      return {
        ...live,
        metadata: { ...live.metadata, resourceVersion: '7' },
      }
    })

    await reconciler.reconcile(makeHost({ spec: { secretRef: 'rotated-secret' } } as never))

    expect(rbacApi.replaceNamespacedRole).toHaveBeenCalledTimes(1)
    const replaced = rbacApi.replaceNamespacedRole.mock.calls[0][0].body
    // The read-only rule includes the mcp-host-runtime-tokens Secret too. Filter by
    // verbs so we don't accidentally pick up a write-only rule.
    const secretRule = replaced.rules.find(
      (r: { resources: string[]; verbs: string[] }) =>
        r.resources[0] === 'secrets' && r.verbs.includes('get')
    )
    expect(secretRule.resourceNames).toEqual([
      'rotated-secret',
      'host-alpha-host-env-secret',
      'host-alpha-host-mcp-host-runtime-tokens',
    ])
    expect(replaced.metadata.resourceVersion).toBe('7')
  })

  it('replaces a live Role that has no rules (fail-open-to-write)', async () => {
    const { reconciler, rbacApi } = createReconciler()
    rbacApi.createNamespacedRole.mockRejectedValueOnce({ code: 409 })
    rbacApi.readNamespacedRole.mockResolvedValue({
      metadata: {
        name: 'host-alpha-host-config-reader',
        namespace: 'mcp-host',
        uid: 'role-uid',
        resourceVersion: '7',
      },
    })

    await reconciler.reconcile(makeHost({ spec: { secretRef: 'rotated-secret' } } as never))

    expect(rbacApi.replaceNamespacedRole).toHaveBeenCalledTimes(1)
    expect(rbacApi.replaceNamespacedRole.mock.calls[0][0].body.metadata.resourceVersion).toBe('7')
  })

  it('cleans up RBAC on reconcileDelete', async () => {
    const { reconciler, coreApi, rbacApi } = createReconciler()
    await reconciler.reconcileDelete('alpha-host', 'mcp-host')

    expect(rbacApi.deleteNamespacedRoleBinding).toHaveBeenCalledWith({
      name: 'host-alpha-host-config-reader',
      namespace: 'mcp-host',
    })
    expect(rbacApi.deleteNamespacedRole).toHaveBeenCalledWith({
      name: 'host-alpha-host-config-reader',
      namespace: 'mcp-host',
    })
    expect(coreApi.deleteNamespacedServiceAccount).toHaveBeenCalledWith({
      name: 'host-alpha-host-sa',
      namespace: 'mcp-host',
    })
  })

  it('treats existing RBAC as success when create returns 409 (idempotent)', async () => {
    const { reconciler, coreApi, rbacApi } = createReconciler()
    coreApi.createNamespacedServiceAccount.mockRejectedValueOnce({ code: 409 })
    rbacApi.createNamespacedRoleBinding.mockRejectedValueOnce({ code: 409 })

    await expect(reconciler.reconcile(makeHost())).resolves.toBeUndefined()
    expect(reconciler.getStatus('alpha-host').deployed).toBe(true)
  })
})

describe('buildChannelReaderDeployment', () => {
  it('returns Deployment with name channel-reader-<host>', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    expect(dep.metadata?.name).toBe('channel-reader-alpha-host')
    expect(dep.metadata?.namespace).toBe('channels')
  })

  it('sets management labels', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost({ name: 'agent-a' }), '')
    // app.kubernetes.io/name MUST be per-Host (channel-reader-<host>) to
    // avoid collision with the static Deployment's selector which matches
    // supersets (see comment in buildChannelReaderDeployment).
    expect(dep.metadata?.labels).toMatchObject({
      app: 'channel-reader',
      'app.kubernetes.io/name': 'channel-reader-agent-a',
      'clerum.io/host': 'agent-a',
      'clerum.io/managed-by': 'host-context-controller',
    })
    expect(dep.spec?.template?.metadata?.labels).toMatchObject({
      app: 'channel-reader',
      'app.kubernetes.io/name': 'channel-reader-agent-a',
      'clerum.io/host': 'agent-a',
    })
  })

  it('sets CLERUM_HOST_REF env to the host name', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost({ name: 'agent-a' }), '')
    const env = dep.spec!.template!.spec!.containers[0].env!
    expect(env).toContainEqual({ name: 'CLERUM_HOST_REF', value: 'agent-a' })
  })

  it('injects CLERUM_NAMESPACE via downward API fieldRef metadata.namespace', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const env = dep.spec!.template!.spec!.containers[0].env!
    expect(env).toContainEqual({
      name: 'CLERUM_NAMESPACE',
      valueFrom: { fieldRef: { fieldPath: 'metadata.namespace' } },
    })
  })

  it('does not inject direct control-api credentials into channel-reader', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const env = dep.spec!.template!.spec!.containers[0].env!
    const envFrom = dep.spec!.template!.spec!.containers[0].envFrom ?? []
    const rendered = JSON.stringify({ env, envFrom })

    expect(rendered).not.toContain('CLERUM_CONTROL_API_SERVICE_TOKEN')
    expect(rendered).not.toContain('CONTROL_API_SERVICE_TOKEN')
    expect(rendered).not.toContain('channel-reader-internal-tokens')
  })

  it('injects CLERUM_MCP_HOST_URL pointing at the slug-scoped mcp-host Service in host.namespace', () => {
    const { reconciler } = createReconciler()
    // mcp-host Service is named host.name and lives in host.namespace
    // (mcp-server here / mcp-host-<slug> in MCC), NOT the channels namespace.
    const dep = (reconciler as any).buildChannelReaderDeployment(
      makeHost({ name: 'agent-a', namespace: 'mcp-host-agent-a' }),
      ''
    )
    const env = dep.spec!.template!.spec!.containers[0].env!
    expect(env).toContainEqual({
      name: 'CLERUM_MCP_HOST_URL',
      value: 'http://agent-a.mcp-host-agent-a.svc.cluster.local:8080',
    })
  })

  it('mounts envFrom on configMapRef clerum-channel-reader-config', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const envFrom = dep.spec!.template!.spec!.containers[0].envFrom!
    expect(envFrom).toContainEqual({ configMapRef: { name: 'clerum-channel-reader-config' } })
  })

  it('sets clerum.io/credentials-revision annotation when revision is provided', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), 'abc123')
    expect(dep.spec?.template?.metadata?.annotations).toMatchObject({
      'clerum.io/credentials-revision': 'abc123',
    })
  })

  it('omits credentials-revision annotation when revision is empty (initial create)', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const ann = dep.spec?.template?.metadata?.annotations ?? {}
    expect(ann['clerum.io/credentials-revision']).toBeUndefined()
  })

  it('runs as non-root with read-only filesystem', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const sc = dep.spec!.template!.spec!.containers[0].securityContext!
    expect(sc).toMatchObject({
      allowPrivilegeEscalation: false,
      readOnlyRootFilesystem: true,
      runAsNonRoot: true,
    })
  })

  it('emits restricted-PSA-compliant container securityContext (drop ALL + seccompProfile)', () => {
    // MCC shared-tenant namespaces (channels-<slug>) are labeled
    // pod-security.kubernetes.io/enforce=restricted. capabilities.drop=[ALL]
    // and seccompProfile=RuntimeDefault are required alongside the existing
    // allowPrivilegeEscalation/runAsNonRoot fields.
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const sc = dep.spec!.template!.spec!.containers[0].securityContext!
    expect(sc).toMatchObject({
      allowPrivilegeEscalation: false,
      readOnlyRootFilesystem: true,
      runAsNonRoot: true,
      runAsUser: 1000,
      runAsGroup: 1000,
      capabilities: { drop: ['ALL'] },
      seccompProfile: { type: 'RuntimeDefault' },
    })
  })

  it('uses imagePullPolicy from config', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    expect(dep.spec!.template!.spec!.containers[0].imagePullPolicy).toBe('IfNotPresent')
  })

  it('keeps internal notification auth separate from mcp-host runtime access', () => {
    // Channel-reader keeps its internal notification token path separate and no
    // longer receives per-host JWT files for direct mcp-host runtime calls.
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    const envNames = (dep.spec?.template?.spec?.containers?.[0]?.env ?? []).map(
      (e: { name: string }) => e.name
    )
    expect(envNames).not.toContain('CLERUM_CHANNEL_READER_BOOTSTRAP_TOKEN')
    expect(envNames).not.toContain('CLERUM_CONTROL_API_URL')
    // No per-host JWT runtime/control token env. The routing URL
    // (CLERUM_MCP_HOST_URL) is allowed: it is a Service address, not auth.
    expect(
      envNames.filter(
        (name: string) => name.startsWith('CLERUM_MCP_HOST_') && name !== 'CLERUM_MCP_HOST_URL'
      )
    ).toEqual([])
  })

  it('sets replicas to 0 when countCommunicationChannels returns 0 (default)', () => {
    const { reconciler } = createReconciler()
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    expect(dep.spec.replicas).toBe(0)
  })

  it('sets replicas to 1 when countCommunicationChannels returns 1+', () => {
    const { reconciler } = createReconciler()
    reconciler.setCountCommunicationChannels(() => 1)
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    expect(dep.spec.replicas).toBe(1)
  })

  it('passes host.name to countCommunicationChannels (not a global sum)', () => {
    // Regression guard: if the count was accidentally summed across hosts,
    // alpha-host would see 5 instead of 3. The replica is still 1 either
    // way, but the per-host count must be passed through faithfully so
    // future metrics (clerum_channel_reader_replicas{host=}) remain
    // host-scoped.
    const { reconciler } = createReconciler()
    const captured: string[] = []
    reconciler.setCountCommunicationChannels(host => {
      captured.push(host)
      return host === 'alpha-host' ? 3 : 5
    })
    ;(reconciler as any).buildChannelReaderDeployment(makeHost(), '')
    expect(captured).toEqual(['alpha-host'])
  })
})

describe('reconcileChannelReaderDeployment', () => {
  it('creates Deployment when absent (createNamespacedDeployment succeeds)', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    appsApi.createNamespacedDeployment.mockResolvedValue({})
    appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))

    expect(appsApi.createNamespacedDeployment).toHaveBeenCalledWith({
      namespace: 'channels',
      body: expect.objectContaining({
        metadata: expect.objectContaining({ name: 'channel-reader-a', namespace: 'channels' }),
      }),
    })
  })

  it('replaces a present Deployment with drift without creating', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: { name: 'channel-reader-a', namespace: 'channels', resourceVersion: '42' },
    })
    appsApi.replaceNamespacedDeployment.mockResolvedValue({})
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))

    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledWith({
      name: 'channel-reader-a',
      namespace: 'channels',
      body: expect.objectContaining({
        metadata: expect.objectContaining({ name: 'channel-reader-a', resourceVersion: '42' }),
      }),
    })
  })

  it('does not replace a converged channel-reader Deployment with server defaults', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    const host = makeHost({ name: 'a' })
    const existing = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    const deploymentSpec = existing.spec
    if (!deploymentSpec?.template.spec) throw new Error('expected channel-reader PodSpec')
    const podSpec = deploymentSpec.template.spec

    existing.metadata = {
      ...existing.metadata,
      resourceVersion: '42',
      uid: 'channel-reader-uid',
      generation: 7,
      creationTimestamp: new Date('2026-07-10T00:00:00Z'),
    }
    existing.status = { readyReplicas: 1, availableReplicas: 1 }
    existing.spec = {
      ...deploymentSpec,
      progressDeadlineSeconds: 600,
      revisionHistoryLimit: 10,
      strategy: {
        type: 'RollingUpdate',
        rollingUpdate: { maxSurge: '25%', maxUnavailable: '25%' },
      },
      template: {
        ...deploymentSpec.template,
        metadata: {
          ...deploymentSpec.template.metadata,
          annotations: { 'kubectl.kubernetes.io/restartedAt': '2026-07-10T00:00:00Z' },
        },
        spec: {
          ...podSpec,
          securityContext: {},
          dnsPolicy: 'ClusterFirst',
          restartPolicy: 'Always',
          schedulerName: 'default-scheduler',
          serviceAccount: podSpec.serviceAccountName,
          terminationGracePeriodSeconds: 30,
          containers: podSpec.containers.map(container => ({
            ...container,
            terminationMessagePath: '/dev/termination-log',
            terminationMessagePolicy: 'File',
            env: container.env?.map(env =>
              env.valueFrom?.fieldRef
                ? {
                    ...env,
                    valueFrom: {
                      ...env.valueFrom,
                      fieldRef: { ...env.valueFrom.fieldRef, apiVersion: 'v1' },
                    },
                  }
                : env
            ),
          })),
        },
      },
    }
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue(existing)
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(host)

    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('replaces a channel-reader Deployment with a non-default strategy', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    const host = makeHost({ name: 'a' })
    const existing = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    existing.metadata = { ...existing.metadata, resourceVersion: '42' }
    existing.spec!.strategy = { type: 'Recreate' }
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue(existing)
    appsApi.replaceNamespacedDeployment.mockResolvedValue({})
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(host)

    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
  })

  it('clears a stale controller-owned revision while preserving operational annotations', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    const host = makeHost({ name: 'a' })
    const existing = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    existing.metadata = { ...existing.metadata, resourceVersion: '42' }
    existing.spec!.template.metadata = {
      ...existing.spec!.template.metadata,
      annotations: {
        'clerum.io/credentials-revision': 'stale-revision',
        'kubectl.kubernetes.io/restartedAt': '2026-07-10T00:00:00Z',
      },
    }
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue(existing)
    appsApi.replaceNamespacedDeployment.mockResolvedValue({})
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(host)

    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
    const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
      .body as k8s.V1Deployment
    expect(
      replacement.spec?.template.metadata?.annotations?.['clerum.io/credentials-revision']
    ).toBeUndefined()
    expect(
      replacement.spec?.template.metadata?.annotations?.['kubectl.kubernetes.io/restartedAt']
    ).toBe('2026-07-10T00:00:00Z')
  })

  it('stops retrying when a fresh channel-reader Deployment converges after a conflict', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    const host = makeHost({ name: 'a' })
    const stale = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    stale.metadata = { ...stale.metadata, resourceVersion: '42' }
    stale.spec!.template.spec!.containers[0].image = 'clerum/channel-reader:stale'
    const converged = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    converged.metadata = { ...converged.metadata, resourceVersion: '43' }

    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(converged)
    appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(host)

    expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
  })

  it('does not overwrite a channel-reader whose ownership changes during a retry', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    const host = makeHost({ name: 'a' })
    const stale = structuredClone(
      (reconciler as any).buildChannelReaderDeployment(host, '')
    ) as k8s.V1Deployment
    stale.metadata = { ...stale.metadata, resourceVersion: '42' }
    stale.spec!.template.spec!.containers[0].image = 'clerum/channel-reader:stale'
    const foreign = structuredClone(stale)
    foreign.metadata = {
      ...foreign.metadata,
      resourceVersion: '43',
      labels: { ...foreign.metadata?.labels, 'clerum.io/host': 'other-host' },
    }

    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(foreign)
    appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await expect((reconciler as any).reconcileChannelReaderDeployment(host)).rejects.toThrow(
      /ownership changed to host "other-host"/
    )

    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
  })

  it('reads existing Secret revision and writes annotation on initial create', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
    appsApi.createNamespacedDeployment.mockResolvedValue({})
    coreApi.readNamespacedSecret.mockResolvedValue({
      data: { 'telegram-bot-token': Buffer.from('tok-1').toString('base64') },
    })
    // Seed the CC cache so computeChannelReaderRevisionForHost finds a Secret ref
    reconciler.setFindCommunicationChannelsByHostRef(() => [
      {
        name: 'cc-a',
        namespace: 'channels',
        spec: { hostRef: 'a', credentialsSecretRef: { name: 'channel-reader-a-credentials' } },
      },
    ])

    await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))

    const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
    const ann = call.body.spec.template.metadata.annotations
    expect(ann['clerum.io/credentials-revision']).toMatch(/^[0-9a-f]{64}$/)
  })

  it('skips reconcile if existing Deployment is owned by a different host', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'channel-reader-a',
        namespace: 'channels',
        labels: { 'clerum.io/host': 'OTHER-HOST' },
      },
    })
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))

    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })

  it('surfaces a transient failure when read-after-409 races with deletion (404)', async () => {
    const { reconciler, appsApi, coreApi } = createReconciler()
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockRejectedValue({ code: 404 })
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
    await expect(
      (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
    ).rejects.toMatchObject({ code: 404 })
    expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
  })
})

describe('reconcile / reconcileDelete with channel-reader', () => {
  it('reconcile(host) calls reconcileChannelReaderDeployment', async () => {
    const { reconciler } = createReconciler()
    const spy = vi
      .spyOn(reconciler as any, 'reconcileChannelReaderDeployment')
      .mockResolvedValue(undefined)
    await reconciler.reconcile(makeHost({ name: 'a' }))
    expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }), expect.any(Function))
  })

  it('reconcileDelete(name, ns) deletes channel-reader-<name> Deployment in channels ns', async () => {
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockResolvedValue({})
    await reconciler.reconcileDelete('a', 'mcp-host')
    expect(appsApi.deleteNamespacedDeployment.mock.calls).toContainEqual([
      { name: 'channel-reader-a', namespace: 'channels' },
    ])
  })

  it('reconcileDelete tolerates 404 on missing channel-reader Deployment', async () => {
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockRejectedValue({ code: 404 })
    await expect(reconciler.reconcileDelete('a', 'mcp-host')).resolves.toBeUndefined()
  })

  it('records channel-reader failure and rejects so the caller can retry convergence', async () => {
    const { reconciler } = createReconciler()
    vi.spyOn(reconciler as any, 'reconcileChannelReaderDeployment').mockRejectedValue(
      new Error('boom')
    )
    await expect(reconciler.reconcile(makeHost({ name: 'a' }))).rejects.toThrow(
      'Failed to converge channel-reader resources for Host "a"'
    )
    expect(reconciler.getStatus('a')).toMatchObject({
      deployed: true,
      ready: true,
      message: expect.stringContaining('channel-reader: boom'),
    })
  })
})

describe('orphan sweep on fullReconcile (channel-reader)', () => {
  it('deletes channel-reader Deployments without a matching Host', async () => {
    // §10.5: an orphan is DISCOVERED via its channel-reader Deployment candidate,
    // its owning Host is confirmed gone by a fresh 404 under known/stable watch
    // authority, and that Host's owned bundle (which cascades to the channels-ns
    // channel-reader Deployment) is deleted. The live Host "a" is retained.
    const getObj = vi.fn().mockRejectedValue({ code: 404 })
    const { reconciler, appsApi } = createReconciler({
      customApi: { getNamespacedCustomObject: getObj },
    })
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
    reconciler.setResolveCurrentHost(name => (name === 'a' ? makeHost({ name }) : undefined))
    // listNamespacedDeployment is called twice during fullReconcile:
    //   1) mcp-host ns (with labelSelector) for the existing host orphan sweep
    //   2) channels ns (no labelSelector) for the channel-reader sweep
    // Mock based on namespace so we don't pollute the host-side sweep.
    appsApi.listNamespacedDeployment.mockImplementation(({ namespace }: { namespace: string }) => {
      if (namespace === 'channels') {
        return Promise.resolve({
          items: [
            {
              metadata: {
                name: 'channel-reader-a',
                namespace: 'channels',
                labels: {
                  app: 'channel-reader',
                  'clerum.io/host': 'a',
                  'clerum.io/managed-by': 'host-context-controller',
                },
              },
            },
            {
              metadata: {
                name: 'channel-reader-orphan',
                namespace: 'channels',
                labels: {
                  app: 'channel-reader',
                  'clerum.io/host': 'orphan',
                  'clerum.io/managed-by': 'host-context-controller',
                },
              },
            },
            { metadata: { name: 'unrelated-thing', namespace: 'channels', labels: {} } },
          ],
        })
      }
      return Promise.resolve({ items: [] })
    })
    appsApi.deleteNamespacedDeployment.mockResolvedValue({})

    // Desired list is empty: "a" is live purely by current-cache presence
    // (setResolveCurrentHost above), which is what §10.5 retention keys on.
    await reconciler.fullReconcile([])

    // The orphan ("orphan") got a fresh authoritative read; the live host ("a")
    // stayed in cache and was never read/deleted.
    expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan' }))
    expect(getObj).not.toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }))
    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
      name: 'channel-reader-orphan',
      namespace: 'channels',
    })
    expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
      name: 'channel-reader-a',
      namespace: 'channels',
    })
    expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
      name: 'unrelated-thing',
      namespace: 'channels',
    })
  })

  it('deletes the owning Host bundle for an orphan discovered via a channel-reader Secret', async () => {
    // The old per-name Secret sweep became discover-owner + delete-the-Host-bundle
    // (§10.5): a managed channel-reader Secret whose owning Host is gone is a
    // DISCOVERY signal. Its owner ("orphan") is confirmed absent by a fresh 404
    // under known/stable authority, and that Host's entire owned bundle is
    // deleted through its per-Host serializer. The live host "a" is retained.
    // (The bundle deletes the Host's own managed Secrets — e.g.
    // `channel-reader-orphan-mcp-host-runtime-auth` — not an arbitrary
    // discovered name, so we assert at the bundle boundary.)
    const getObj = vi.fn().mockRejectedValue({ code: 404 })
    const { reconciler, coreApi } = createReconciler({
      customApi: { getNamespacedCustomObject: getObj },
    })
    reconciler.setHostWatchAuthority(() => ({ known: true, generation: 1 }))
    reconciler.setResolveCurrentHost(name => (name === 'a' ? makeHost({ name }) : undefined))
    coreApi.listNamespacedSecret.mockResolvedValue({
      items: [
        {
          metadata: {
            name: 'channel-reader-a-credentials',
            namespace: 'channels',
            labels: {
              'clerum.io/component': 'channel-reader',
              'clerum.io/host': 'a',
              'clerum.io/managed-by': 'host-context-controller',
            },
          },
        },
        {
          metadata: {
            name: 'channel-reader-orphan-credentials',
            namespace: 'channels',
            labels: {
              'clerum.io/component': 'channel-reader',
              'clerum.io/host': 'orphan',
              'clerum.io/managed-by': 'host-context-controller',
            },
          },
        },
      ],
    })
    const deleteBundle = vi
      .spyOn(reconciler as any, 'deleteHostRuntimeResources')
      .mockResolvedValue(undefined)

    // Desired list is empty: "a" is live purely by current-cache presence
    // (setResolveCurrentHost above), which is what §10.5 retention keys on.
    await reconciler.fullReconcile([])

    expect(getObj).toHaveBeenCalledWith(expect.objectContaining({ name: 'orphan' }))
    expect(deleteBundle).toHaveBeenCalledWith('orphan', 'mcp-host')
    expect(deleteBundle).not.toHaveBeenCalledWith('a', 'mcp-host')
  })

  it('defers the orphan bundle and deletes nothing when watch authority is unknown', async () => {
    // §10.5 gate proven on purpose: unknown watch authority short-circuits the
    // whole cleanup BEFORE any candidate is gathered or any fresh read is made,
    // so nothing is deleted even though an orphan channel-reader Deployment is
    // present. This is the accidental safety the removed global tail used to
    // provide, now made explicit.
    const getObj = vi.fn()
    const { reconciler, appsApi } = createReconciler({
      customApi: { getNamespacedCustomObject: getObj },
    })
    reconciler.setHostWatchAuthority(() => ({ known: false, generation: 1 }))
    reconciler.setResolveCurrentHost(() => undefined)
    appsApi.listNamespacedDeployment.mockImplementation(({ namespace }: { namespace: string }) =>
      namespace === 'channels'
        ? Promise.resolve({
            items: [
              {
                metadata: {
                  name: 'channel-reader-orphan',
                  namespace: 'channels',
                  labels: {
                    app: 'channel-reader',
                    'clerum.io/host': 'orphan',
                    'clerum.io/managed-by': 'host-context-controller',
                  },
                },
              },
            ],
          })
        : Promise.resolve({ items: [] })
    )
    const deleteBundle = vi
      .spyOn(reconciler as any, 'deleteHostRuntimeResources')
      .mockResolvedValue(undefined)
    const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)

    await reconciler.fullReconcile([])

    expect(getObj).not.toHaveBeenCalled()
    expect(deleteBundle).not.toHaveBeenCalled()
    expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalledWith({
      name: 'channel-reader-orphan',
      namespace: 'channels',
    })
    expect(warn).toHaveBeenCalledWith('Deferring orphan cleanup', { reason: 'authority_unknown' })
    warn.mockRestore()
  })

  it('does NOT delete Secrets missing the clerum.io/managed-by label', async () => {
    const { reconciler, coreApi } = createReconciler()
    coreApi.listNamespacedSecret.mockResolvedValue({
      items: [
        {
          metadata: {
            name: 'hand-rolled-channel-secret',
            namespace: 'channels',
            labels: {
              'clerum.io/component': 'channel-reader',
              'clerum.io/host': 'unknown-host',
              // intentionally NO managed-by
            },
          },
        },
      ],
    })
    coreApi.deleteNamespacedSecret.mockResolvedValue({})
    await reconciler.fullReconcile([])
    expect(coreApi.deleteNamespacedSecret).not.toHaveBeenCalledWith({
      name: 'hand-rolled-channel-secret',
      namespace: 'channels',
    })
  })
})

describe('sweepLegacyStaticChannelReader', () => {
  // Issue #273: HCC startup sweep that retires the static
  // `clerum-channel-reader` Deployment so per-Host pods don't collide
  // with it on Telegram getUpdates long-poll (409 Conflict).
  // Empirical reproduction in #273 comment.

  it('deletes the static clerum-channel-reader Deployment when it exists', async () => {
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockResolvedValue({})

    await reconciler.sweepLegacyStaticChannelReader()

    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
      name: 'clerum-channel-reader',
      namespace: 'channels',
    })
  })

  it('swallows 404 (already gone — steady state)', async () => {
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockRejectedValue(
      Object.assign(new Error('not found'), { code: 404 })
    )

    // Must not throw — 404 is the steady-state after the legacy retirement.
    await expect(reconciler.sweepLegacyStaticChannelReader()).resolves.toBeUndefined()
    expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledTimes(1)
  })

  it('logs but does not throw on non-404 errors (e.g. 403 RBAC)', async () => {
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockRejectedValue(
      Object.assign(new Error('forbidden'), { code: 403 })
    )
    const errorSpy = vi.spyOn(HostContextLogger.prototype, 'error').mockImplementation(() => {})

    // Reviewer-style assertion: startup MUST continue even if the sweep
    // fails — per-Host reconciles are still required.
    await expect(reconciler.sweepLegacyStaticChannelReader()).resolves.toBeUndefined()
    expect(errorSpy).toHaveBeenCalledWith(
      expect.stringContaining('legacy-sweep failed'),
      expect.objectContaining({ err: expect.anything() })
    )

    errorSpy.mockRestore()
  })

  it('targets the channels namespace (not mcp-host) and the exact name', async () => {
    // Regression guard: if someone refactors namespace config, the sweep
    // must not accidentally target a Deployment named `clerum-channel-reader`
    // in a different namespace.
    const { reconciler, appsApi } = createReconciler()
    appsApi.deleteNamespacedDeployment.mockResolvedValue({})

    await reconciler.sweepLegacyStaticChannelReader()

    const call = appsApi.deleteNamespacedDeployment.mock.calls[0][0] as {
      name: string
      namespace: string
    }
    expect(call.name).toBe('clerum-channel-reader')
    expect(call.namespace).toBe('channels')
  })
})

describe('setCountCommunicationChannels', () => {
  it('defaults to a function returning 0 when not set', () => {
    const { reconciler } = createReconciler()
    expect((reconciler as any).countCommunicationChannels('any-host')).toBe(0)
  })

  it('uses the injected callback to count CCs for a host', () => {
    const { reconciler } = createReconciler()
    const counts = new Map([
      ['alpha-host', 3],
      ['beta-host', 0],
    ])
    reconciler.setCountCommunicationChannels(host => counts.get(host) ?? 0)
    expect((reconciler as any).countCommunicationChannels('alpha-host')).toBe(3)
    expect((reconciler as any).countCommunicationChannels('beta-host')).toBe(0)
    expect((reconciler as any).countCommunicationChannels('unknown-host')).toBe(0)
  })
})

describe('resolveWorkflowControlScopes (F25)', () => {
  const DEFAULT: HostWorkflowControlScope[] = [
    'workflow:list',
    'workflow:read',
    'workflow:trigger',
    'workflow:approval:resolve',
    'workflow:approval:decide',
  ]

  it('exports the canonical first-party scope set for explicit declarations', () => {
    expect(DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES).toEqual(DEFAULT)
  })

  it('does not grant workflow-control scopes when workflowControl is absent without channel ingress', () => {
    expect(resolveWorkflowControlScopes(undefined)).toEqual([])
  })

  it('defaults first-party workflow-control scopes when a channel-bearing Host omits workflowControl', () => {
    expect(resolveWorkflowControlScopes(undefined, { hasChannelIngress: true })).toEqual(DEFAULT)
  })

  it('does not grant workflow-control scopes when the block is present but scopes is omitted', () => {
    expect(resolveWorkflowControlScopes({})).toEqual([])
  })

  it('honors an explicit empty scopes list as an intentional opt-out (does NOT default)', () => {
    expect(resolveWorkflowControlScopes({ scopes: [] })).toEqual([])
  })

  it('honors an explicit scopes list exactly as declared', () => {
    expect(resolveWorkflowControlScopes({ scopes: ['workflow:list', 'workflow:read'] })).toEqual([
      'workflow:list',
      'workflow:read',
    ])
  })

  it('returns a fresh array (callers may sort/mutate without corrupting the default)', () => {
    const a = resolveWorkflowControlScopes({ scopes: DEFAULT })
    a.sort()
    expect(DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES).toEqual(DEFAULT)
  })
})

// ── B2: CC cache-sync gate on replica decision ─────────────────────────────
describe('buildChannelReaderDeployment — B2 CC cache-sync gate', () => {
  it('sets replicas to 1 when cache is synced and CC count is 1', () => {
    // When the CC cache has completed its initial list and the host has 1+ CCs,
    // the Deployment should be created with replicas=1.
    const { reconciler } = createReconciler()
    reconciler.setCountCommunicationChannels(() => 1)
    reconciler.setIsCommunicationChannelCacheSynced(() => true)
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '', undefined)
    expect(dep.spec.replicas).toBe(1)
  })

  it('preserves replicas=1 when cache is NOT synced and an existing Deployment has replicas=1', () => {
    // When the CC cache is still empty (startup / load failure) AND we know the
    // existing Deployment had replicas=1, we MUST NOT scale to 0 — the channel-reader
    // is live and scaling it down would drop inbound messages.
    const { reconciler } = createReconciler()
    // ccCacheSyncedFn returns false (unsynced)
    reconciler.setIsCommunicationChannelCacheSynced(() => false)
    // countCommunicationChannels returns 0 (cache empty)
    reconciler.setCountCommunicationChannels(() => 0)
    const dep = (reconciler as any).buildChannelReaderDeployment(makeHost(), '', 1)
    expect(dep.spec.replicas).toBe(1)
  })
})

describe('reconcileChannelReaderDeployment — B2: preserve replicas on unsynced cache (409 path)', () => {
  it('keeps replicas=1 on the replace body when cache is unsynced and existing Deployment has replicas=1', async () => {
    // Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced).
    // The existing live Deployment has replicas=1 (channel-reader is running).
    // reconcileChannelReaderDeployment must NOT scale it to 0.
    const { reconciler, appsApi, coreApi } = createReconciler()

    // Force the 409 → read-existing → replace path
    appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
    appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        name: 'channel-reader-a',
        namespace: 'channels',
        resourceVersion: '77',
        labels: { 'clerum.io/host': 'a' },
      },
      spec: { replicas: 1 },
    })
    appsApi.replaceNamespacedDeployment.mockResolvedValue({})
    coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })

    // CC cache is unsynced; count returns 0
    reconciler.setIsCommunicationChannelCacheSynced(() => false)
    reconciler.setCountCommunicationChannels(() => 0)

    await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))

    // The replace call body must carry replicas=1, not 0
    expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
    const replaceBody = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
      .body as k8s.V1Deployment
    expect(replaceBody.spec?.replicas).toBe(1)
  })
})

async function readFleetBenignSupersessions(errorName: string): Promise<number> {
  const metric = registry.getSingleMetric('clerum_hcc_host_fleet_benign_supersessions_total')
  if (!metric) throw new Error('clerum_hcc_host_fleet_benign_supersessions_total is not registered')
  const snapshot = await metric.get()
  return snapshot.values.find(entry => entry.labels.error === errorName)?.value ?? 0
}

describe('collectHostReconcileFailures benign supersession (#490)', () => {
  function nameEquivalent(name: string, message: string): Error {
    const error = new Error(message)
    error.name = name
    return error
  }

  it.each([
    'HostInventoryAuthorityUnavailableError',
    'HostMutationIdentityChangedError',
    'HostMutationSpecRevisionChangedError',
    'HostMutationDependencyChangedError',
  ] as const)(
    'resolves reconcileHosts when every fleet error is name-equivalent %s',
    async errorName => {
      const { reconciler } = createReconciler()
      const withdrawn = nameEquivalent(
        errorName,
        `Host "${errorName}" superseded before Host "alpha-host" reconcile admission`
      )
      vi.spyOn(reconciler, 'reconcile').mockRejectedValue(withdrawn)
      const errorSpy = vi.spyOn(HostContextLogger.prototype, 'error').mockImplementation(() => {})
      const before = await readFleetBenignSupersessions(errorName)

      await expect(reconciler.reconcileHosts([makeHost()])).resolves.toBeUndefined()

      expect(await readFleetBenignSupersessions(errorName)).toBe(before + 1)
      expect(
        errorSpy.mock.calls.some(call => String(call[0]).includes('Fleet reconcile failed'))
      ).toBe(false)
      errorSpy.mockRestore()
    }
  )

  it('still rejects a lookalike error name that is not in the benign set', async () => {
    const { reconciler } = createReconciler()
    vi.spyOn(reconciler, 'reconcile').mockRejectedValue(
      nameEquivalent('HostMutationSpecRevisionChanged', 'almost the withdrawn name')
    )

    await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(
      HostFleetReconcileError
    )
  })

  it('still rejects HostFleetReconcileError for a non-benign fleet error', async () => {
    const { reconciler } = createReconciler()
    vi.spyOn(reconciler, 'reconcile').mockRejectedValue(new Error('boom'))

    await expect(reconciler.reconcileHosts([makeHost()])).rejects.toBeInstanceOf(
      HostFleetReconcileError
    )
  })

  it('still rejects when a mixed pass has at least one genuine host failure', async () => {
    const { reconciler } = createReconciler()
    vi.spyOn(reconciler, 'reconcile').mockImplementation(async host => {
      if (host.name === 'alpha-host') {
        throw nameEquivalent(
          'HostMutationSpecRevisionChangedError',
          'Host spec generation changed before Host "alpha-host" reconcile admission'
        )
      }
      throw new Error('boom')
    })

    await expect(
      reconciler.reconcileHosts([makeHost(), makeHost({ name: 'beta-host' })])
    ).rejects.toMatchObject({
      name: 'HostFleetReconcileError',
      hostFailures: [expect.objectContaining({ message: 'boom' })],
    })
  })
})

describe('HostReconciler oauth:user-token runtime scope provisioning', () => {
  // Extract the runtime-token scope arg from the LAST issuer call, sorted for
  // stable comparison. This is the REAL derive → issuance payload: the mock
  // captures whatever `resolveWorkflowControlScopesForHost` (the production
  // derive) actually produced — no hand-minted JWT.
  function lastIssuedScopes(): string[] {
    const calls = vi.mocked(issueMcpHostRuntimeTokens).mock.calls
    const last = calls[calls.length - 1]
    return [...(last?.[2] ?? [])].sort()
  }

  // Access the private static scope-hash used for drift detection.
  const scopeHashOf = (
    HostReconciler as unknown as {
      runtimeTokenScopeHash(
        host: HostCRD,
        hasChannelIngress?: boolean,
        frontsOAuthServer?: boolean,
        projection?: { derivedScopes?: string[]; driftHashInput?: string },
        grokProjection?: { derivedScopes?: string[]; driftHashInput?: string }
      ): string
    }
  ).runtimeTokenScopeHash

  it('keeps the pre-Grok scope hash when Grok has no derived scopes', () => {
    const host = makeHost()
    const codex = { derivedScopes: ['llm:codex:execute'], driftHashInput: '{"k":1}' }
    expect(scopeHashOf(host, false, false, codex, { derivedScopes: [] })).toBe(
      scopeHashOf(host, false, false, codex)
    )
    expect(
      scopeHashOf(host, false, false, codex, { derivedScopes: ['llm:grok:execute'] })
    ).not.toBe(scopeHashOf(host, false, false, codex))
  })

  it('requests oauth:user-token through the real issuance payload when the Host fronts an enabled oauth mcp-server', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const { reconciler } = createReconciler()
    // Host CRD carries NO oauth scope — the value is derive-only. HCC learns
    // the Host fronts an oauth server exclusively from this injected probe.
    reconciler.setHostFrontsOAuthServer(async () => true)

    await reconciler.reconcile(makeHost())

    // Goes through the REAL derive (resolveWorkflowControlScopesForHost →
    // resolveRuntimeControlScopes). Without the derive change this scope is
    // absent and the assertion fails.
    expect(lastIssuedScopes()).toEqual(
      [...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES, OAUTH_USER_TOKEN_SCOPE].sort()
    )
    expect(lastIssuedScopes()).toContain('oauth:user-token')
  })

  it('does NOT request oauth:user-token when the Host fronts no oauth mcp-server', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const { reconciler } = createReconciler()
    reconciler.setHostFrontsOAuthServer(async () => false)

    await reconciler.reconcile(makeHost())

    expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
    expect(lastIssuedScopes()).not.toContain('oauth:user-token')
  })

  it('defaults to NO oauth:user-token when the probe is unwired', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const { reconciler } = createReconciler()
    // No setHostFrontsOAuthServer — production default is fail-closed false.

    await reconciler.reconcile(makeHost())

    expect(lastIssuedScopes()).not.toContain('oauth:user-token')
  })

  it('fails closed (no oauth:user-token) when the oauth-server probe throws', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')
    try {
      const { reconciler } = createReconciler()
      reconciler.setHostFrontsOAuthServer(async () => {
        throw new Error('context read failed')
      })

      await reconciler.reconcile(makeHost())

      expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
      expect(lastIssuedScopes()).not.toContain('oauth:user-token')
      // A new Host has no retained grant to preserve, so the unknown answer
      // fails closed at issuance, and says so.
      const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))
      expect(retryWarn?.[1]).toMatchObject({ attempts: 3, err: 'context read failed' })
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('toggling oauth-server presence changes the runtime-token scope hash (drift re-issues)', () => {
    const host = makeHost()
    const withoutOAuth = scopeHashOf(host, true, false)
    const withOAuth = scopeHashOf(host, true, true)
    expect(withOAuth).not.toBe(withoutOAuth)
  })

  it('mint scope set matches the scope-hash input (no mint/hash divergence)', async () => {
    vi.mocked(issueMcpHostRuntimeTokens).mockClear()
    const { reconciler } = createReconciler()
    reconciler.setHostFrontsOAuthServer(async () => true)
    const host = makeHost()

    await reconciler.reconcile(host)

    // The scopes minted into the token (issuer payload) must equal the exact
    // set the drift hash is computed over — both flow through
    // resolveRuntimeControlScopes, so a hash miss would re-issue forever.
    const hashInput = [
      ...resolveRuntimeControlScopes(host.spec.workflowControl, {
        hasChannelIngress: true,
        frontsOAuthServer: true,
      }),
    ].sort()
    expect(lastIssuedScopes()).toEqual(hashInput)
    expect(hashInput).toContain('oauth:user-token')
  })
})

describe('HostReconciler.observeFrontsOAuthServer transient-blip retry (R4-M9)', () => {
  // The oauth-server probe is exercised through the REAL reconciler built by
  // createReconciler() and the REAL setHostFrontsOAuthServer setter — no
  // hand-minted value. We assert on the private observation directly (rather
  // than the full reconcile) on purpose: the deployment drift guard
  // (ensureCurrentRuntimeTokenScope) re-reads the observation mid-reconcile, so
  // a probe that throws once and then returns `true` would be MASKED at the
  // public issuance observable. The observation IS the value whose flip drives
  // the churn this fixes; what an unobserved answer means is decided by its
  // callers and covered at the reconcile level.
  const observeFronts = (
    reconciler: HostReconciler,
    host: HostCRD
  ): Promise<
    | { observed: true; frontsOAuthServer: boolean }
    | { observed: false; attempts: number; error: unknown }
  > =>
    (
      reconciler as unknown as {
        observeFrontsOAuthServer(
          host: HostCRD
        ): Promise<
          | { observed: true; frontsOAuthServer: boolean }
          | { observed: false; attempts: number; error: unknown }
        >
      }
    ).observeFrontsOAuthServer(host)

  it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {
    // Regression case (T3): without the bounded retry the throw is reported as
    // unobserved and this assertion fails. With the retry the second attempt
    // returns the authoritative `true`.
    const { reconciler } = createReconciler()
    let calls = 0
    reconciler.setHostFrontsOAuthServer(async () => {
      calls += 1
      if (calls === 1) throw new Error('transient apiserver read blip')
      return true
    })

    await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
      observed: true,
      frontsOAuthServer: true,
    })
    expect(calls).toBe(2)
  })

  it('does NOT retry an authoritative false: probe returns false without throwing → false, called once', async () => {
    // Control: a real scope change RETURNS false; the retry must not fire, or a
    // genuine "no longer oauth" would be smeared into a stale `true`.
    const { reconciler } = createReconciler()
    let calls = 0
    reconciler.setHostFrontsOAuthServer(async () => {
      calls += 1
      return false
    })

    await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
      observed: true,
      frontsOAuthServer: false,
    })
    expect(calls).toBe(1)
  })

  it('reports an unobserved answer after exhausting retries instead of collapsing it to false', async () => {
    const { reconciler } = createReconciler()
    let calls = 0
    const outage = new Error('sustained apiserver outage')
    reconciler.setHostFrontsOAuthServer(async () => {
      calls += 1
      throw outage
    })

    await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
      observed: false,
      attempts: 3,
      error: outage,
    })
    // Retried up to the bounded ceiling (3 total attempts).
    expect(calls).toBe(3)
  })
})