Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
47.2 KB
Raw Blame
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import {
  type SharedFileSystemFactoryConfig,
  WFC_MOUNT_PATH,
  buildDeployment,
  buildEgressNetworkPolicy,
  buildIngressNetworkPolicy,
  buildPvc,
  buildService,
  pvcName,
  wfcDeploymentName,
  wfcEgressPolicyName,
  wfcIngressPolicyName,
  wfcInitJobNamePrefix,
  wfcServiceName,
} from '../k8s/sharedFileSystemFactory'
import { createsTotal, existenceReadsTotal } from '../metrics'
import { SharedFileSystemReconciler } from '../sharedFileSystemReconciler'
import type { SharedFileSystemCRD } from '../types'

const factoryConfig: SharedFileSystemFactoryConfig = {
  hostNamespace: 'mcp-host',
  controlPlaneNamespace: 'control-plane',
  wfcImage: 'registry.example/clerum/workspace-files-controller:0.1.0',
  wfcImagePullPolicy: 'IfNotPresent',
  wfcImagePullSecretName: 'clerum',
  wfcPort: 8086,
  wfcInitImage: 'busybox:1.36',
  wfcResources: {
    requests: { memory: '64Mi', cpu: '50m' },
    limits: { memory: '128Mi', cpu: '200m' },
  },
  wfcJwtPublicKeyConfigMapName: 'mcp-host-config',
  wfcJwtPublicKeyConfigMapKey: 'CLERUM_AUTH_JWT_PUBLIC_KEY',
  wfcMaxUploadBytes: 100 * 1024 * 1024,
  wfcMaxListEntries: 5000,
  wfcMaxPathDepth: 32,
}

function makeSfs(overrides: Partial<SharedFileSystemCRD> = {}): SharedFileSystemCRD {
  return {
    name: 'team-mission',
    namespace: 'mcp-host',
    spec: {
      size: '5Gi',
      directories: ['docs', 'runbooks'],
      security: { runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000 },
    },
    ...overrides,
  }
}

function err(code: number, msg = 'k8s error'): Error & { code: number } {
  return Object.assign(new Error(msg), { code })
}
const notFound = () => err(404, 'not found')
const alreadyExists = () => err(409, 'already exists')

function liveDeployment(
  sfs = makeSfs(),
  status: k8s.V1DeploymentStatus = { readyReplicas: 1, availableReplicas: 1 }
): k8s.V1Deployment {
  const desired = buildDeployment(sfs, factoryConfig)
  return { ...desired, metadata: { ...desired.metadata, resourceVersion: '1' }, status }
}

function livePvc(
  sfs = makeSfs(),
  status: k8s.V1PersistentVolumeClaimStatus = { phase: 'Bound', capacity: { storage: '5Gi' } }
): k8s.V1PersistentVolumeClaim {
  const desired = buildPvc(sfs, factoryConfig)
  return { ...desired, metadata: { ...desired.metadata, resourceVersion: '1' }, status }
}

function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {
  // Status/cleanup tests use already-converged policies with real identity and
  // ownership. Cold-start tests explicitly arrange absence for their own reads.
  const policies = systems.flatMap(sfs => [
    buildIngressNetworkPolicy(sfs, factoryConfig),
    buildEgressNetworkPolicy(sfs, factoryConfig),
  ])
  const appsApi = {
    createNamespacedDeployment: vi.fn().mockResolvedValue({}),
    // Existence and readiness share an actual named Deployment; status tests
    // vary its status without erasing its desired spec or ownership metadata.
    readNamespacedDeployment: vi.fn(
      async ({ name }: { name: string }): Promise<k8s.V1Deployment> => {
        const sfs = systems.find(item => wfcDeploymentName(item) === name)
        if (!sfs) throw new Error(`Missing Deployment fixture for ${name}`)
        return liveDeployment(sfs)
      }
    ),
    replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),
    deleteNamespacedDeployment: vi.fn().mockResolvedValue({}),
  }
  const coreApi = {
    createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
    deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
    createNamespacedService: vi.fn().mockResolvedValue({}),
    readNamespacedService: vi.fn(async ({ name }: { name: string }): Promise<k8s.V1Service> => {
      const sfs = systems.find(item => wfcServiceName(item) === name)
      if (!sfs) throw new Error(`Missing Service fixture for ${name}`)
      const desired = buildService(sfs, factoryConfig)
      return {
        ...desired,
        metadata: { ...desired.metadata, resourceVersion: '1' },
        spec: { ...desired.spec, clusterIP: '10.96.1.23' },
      }
    }),
    replaceNamespacedService: vi.fn(),
    deleteNamespacedService: vi.fn().mockResolvedValue({}),
    // Truthful status (#592): assessReadiness reads the PVC bind state, and
    // detectWfcWedge lists the wfc Pod. Defaults = happy path (PVC Bound, no
    // wedged pod) so existing tests reach phase=Ready.
    readNamespacedPersistentVolumeClaim: vi.fn(
      async ({ name }: { name: string }): Promise<k8s.V1PersistentVolumeClaim> => {
        const sfs = systems.find(item => pvcName(item) === name)
        if (!sfs) throw new Error(`Missing PVC fixture for ${name}`)
        return livePvc(sfs)
      }
    ),
    replaceNamespacedPersistentVolumeClaim: vi.fn(),
    listNamespacedPod: vi.fn().mockResolvedValue({ items: [] }),
  }
  const batchApi = {
    // Legacy init-Job cleanup surface (replaces createNamespacedJob).
    listNamespacedJob: vi.fn().mockResolvedValue({ items: [] }),
    deleteNamespacedJob: vi.fn().mockResolvedValue({}),
    createNamespacedJob: vi.fn().mockResolvedValue({}), // must NEVER be called now
  }
  const networkingApi = {
    createNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    readNamespacedNetworkPolicy: vi.fn(async ({ name }: { name: string }) => {
      const policy = policies.find(item => item.metadata?.name === name)
      if (!policy) throw new Error(`Missing NetworkPolicy fixture for ${name}`)
      return { ...policy, metadata: { ...policy.metadata, resourceVersion: '1' } }
    }),
    replaceNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
    deleteNamespacedNetworkPolicy: vi.fn().mockResolvedValue({}),
  }
  const customApi = {
    patchNamespacedCustomObjectStatus: vi.fn().mockResolvedValue({}),
  }
  return { appsApi, coreApi, batchApi, networkingApi, customApi }
}

function makeReconciler(mocks: ReturnType<typeof makeMocks>, now?: () => Date) {
  return new SharedFileSystemReconciler(null, {
    /* eslint-disable @typescript-eslint/no-explicit-any */
    appsApi: mocks.appsApi as any,
    coreApi: mocks.coreApi as any,
    batchApi: mocks.batchApi as any,
    networkingApi: mocks.networkingApi as any,
    customApi: mocks.customApi as any,
    /* eslint-enable @typescript-eslint/no-explicit-any */
    now: now ?? (() => new Date('2026-04-30T12:00:00Z')),
    factoryConfig,
  })
}

/** A legacy init Job manifest, optionally with a mismatching name/label. */
function legacyJob(sfs: SharedFileSystemCRD, name?: string) {
  return {
    metadata: {
      name: name ?? `${wfcInitJobNamePrefix(sfs)}g0001`,
      labels: {
        'clerum.io/sharedfilesystem': sfs.name,
        'clerum.io/sharedfilesystem-namespace': sfs.namespace,
      },
    },
  }
}

describe('SharedFileSystemReconciler — monotonic status on re-reconcile (#592 race)', () => {
  it('does not transiently downgrade an already-Ready SFS below Ready while re-reconciling', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    const sfs = makeSfs()

    // First reconcile reaches Ready (happy-path mocks: PVC Bound, wfc ready).
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')

    // Capture the in-memory phase observed MID-reconcile — this is the exact
    // window resolveContextMounts() reads to gate the RO mount. The earlier bug
    // wrote phase=Provisioning (before ensurePvc) and phase=Initializing (before
    // ensureService) unconditionally, so a concurrent Host reconcile saw a
    // not-Ready SFS and deferred the mount forever. ensurePvc + ensureService run
    // before assessReadiness restores the truthful phase, so they are the probes.
    const phasesSeenMidReconcile: Array<string | undefined> = []
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockImplementation(async () => {
      phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)
      return livePvc(sfs)
    })
    mocks.coreApi.readNamespacedService.mockImplementation(async () => {
      phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)
      return buildService(sfs, factoryConfig)
    })

    // Re-reconcile the already-Ready SFS (mirrors a Context-change re-reconcile).
    await reconciler.reconcile(sfs)

    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
    expect(phasesSeenMidReconcile).toHaveLength(3) // PVC existence + Service existence + PVC readiness
    for (const phase of phasesSeenMidReconcile) {
      expect(phase).toBe('Ready')
    }
  })
})

describe('SharedFileSystemReconciler — happy path', () => {
  let mocks: ReturnType<typeof makeMocks>
  let reconciler: SharedFileSystemReconciler

  beforeEach(() => {
    mocks = makeMocks()
    reconciler = makeReconciler(mocks)
  })

  it('creates PVC, Service, Deployment, and both NetworkPolicies — and NEVER an init Job', async () => {
    const sfs = makeSfs()
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(notFound())
    mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
    mocks.coreApi.readNamespacedService.mockRejectedValueOnce(notFound())
    mocks.networkingApi.readNamespacedNetworkPolicy
      .mockRejectedValueOnce(notFound())
      .mockRejectedValueOnce(notFound())
    await reconciler.reconcile(sfs)

    expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
    expect(mocks.coreApi.createNamespacedService).toHaveBeenCalledTimes(1)
    expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledWith(
      expect.objectContaining({
        body: expect.objectContaining({
          metadata: expect.objectContaining({ name: wfcDeploymentName(sfs) }),
        }),
      })
    )
    expect(mocks.networkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
    // The separate init Job is gone; seeding is the Deployment's initContainer.
    expect(mocks.batchApi.createNamespacedJob).not.toHaveBeenCalled()
  })

  it('builds the Deployment with a root initContainer that seeds the PVC in the same pod', async () => {
    const sfs = makeSfs()
    mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
    await reconciler.reconcile(sfs)
    const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body
    const podSpec = dep.spec.template.spec
    expect(podSpec.initContainers).toHaveLength(1)
    expect(podSpec.initContainers[0].name).toBe('init')
    expect(podSpec.initContainers[0].securityContext.runAsUser).toBe(0)
    expect(podSpec.initContainers[0].securityContext.runAsNonRoot).toBe(false)
    expect(podSpec.containers).toHaveLength(1)
  })

  it('writes status with phase=Ready and does NOT fabricate a lastInitJob', async () => {
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    const call = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.at(-1)![0]
    expect(call.body[0]).toMatchObject({ op: 'replace', path: '/status' })
    expect(call.body[0].value).toMatchObject({
      phase: 'Ready',
      pvcName: pvcName(sfs),
      serviceName: wfcServiceName(sfs),
    })
    expect(call.body[0].value.lastInitJob).toBeUndefined()
  })

  it('reaches phase=Ready after the wfc resources are applied', async () => {
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
  })
})

describe('SharedFileSystemReconciler — legacy init-Job cleanup', () => {
  it('lists Jobs by the SFS label and deletes only those matching the wfc-init-<hash>- prefix', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    mocks.batchApi.listNamespacedJob.mockResolvedValue({
      items: [
        legacyJob(sfs), // our init Job → delete
        legacyJob(sfs, 'some-other-job'), // shares label, wrong prefix → KEEP
      ],
    })
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcile(sfs)

    expect(mocks.batchApi.listNamespacedJob).toHaveBeenCalledWith(
      expect.objectContaining({
        namespace: 'mcp-host',
        // Clear scoping by the SFS's own name + namespace labels.
        labelSelector: `clerum.io/sharedfilesystem=${sfs.name},clerum.io/sharedfilesystem-namespace=${sfs.namespace}`,
      })
    )
    expect(mocks.batchApi.deleteNamespacedJob).toHaveBeenCalledTimes(1)
    expect(mocks.batchApi.deleteNamespacedJob).toHaveBeenCalledWith(
      expect.objectContaining({ name: `${wfcInitJobNamePrefix(sfs)}g0001` })
    )
  })

  it('never deletes a foreign Job that merely shares the label (double guard)', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    mocks.batchApi.listNamespacedJob.mockResolvedValue({
      items: [legacyJob(sfs, wfcDeploymentName(sfs)), legacyJob(sfs, 'unrelated')],
    })
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcile(sfs)
    expect(mocks.batchApi.deleteNamespacedJob).not.toHaveBeenCalled()
  })

  it('is non-fatal: a cleanup list failure does NOT block the Deployment rewrite', async () => {
    const mocks = makeMocks()
    mocks.batchApi.listNamespacedJob.mockRejectedValue(err(500, 'list boom'))
    const drifted = liveDeployment()
    drifted.spec!.replicas = 9
    mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce(drifted)
    const reconciler = makeReconciler(mocks)
    await expect(reconciler.reconcile(makeSfs())).resolves.toBeUndefined()
    expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(reconciler.getStatus(makeSfs()).phase).toBe('Ready')
  })

  it('swallows 404/409 on individual Job deletes', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    mocks.batchApi.listNamespacedJob.mockResolvedValue({
      items: [
        legacyJob(sfs, `${wfcInitJobNamePrefix(sfs)}a`),
        legacyJob(sfs, `${wfcInitJobNamePrefix(sfs)}b`),
      ],
    })
    mocks.batchApi.deleteNamespacedJob
      .mockRejectedValueOnce(notFound())
      .mockRejectedValueOnce(alreadyExists())
    const reconciler = makeReconciler(mocks)
    await expect(reconciler.reconcile(sfs)).resolves.toBeUndefined()
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
  })

  it('requests Background propagation so the Job pod is reaped too', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    mocks.batchApi.listNamespacedJob.mockResolvedValue({ items: [legacyJob(sfs)] })
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcile(sfs)
    expect(mocks.batchApi.deleteNamespacedJob).toHaveBeenCalledWith(
      expect.objectContaining({ propagationPolicy: 'Background' })
    )
  })
})

describe('SharedFileSystemReconciler — idempotence', () => {
  it('leaves existing equivalent NetworkPolicies in place without POST or PUT', async () => {
    const mocks = makeMocks()
    await makeReconciler(mocks).reconcile(makeSfs())
    expect(mocks.networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
    expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })

  it('propagates a denied NetworkPolicy read without attempting creation', async () => {
    const mocks = makeMocks()
    const denied = err(403, 'policy read denied')
    mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce(denied)
    await expect(makeReconciler(mocks).reconcile(makeSfs())).rejects.toBe(denied)
    expect(mocks.networkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
    expect(mocks.networkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
  })

  it('PVC GET404/create409 rereads the concurrent object without replacing it', async () => {
    const mocks = makeMocks()
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(notFound())
    mocks.coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce(alreadyExists())
    const reconciler = makeReconciler(mocks)
    await expect(reconciler.reconcile(makeSfs())).resolves.toBeUndefined()
    expect(reconciler.getStatus(makeSfs()).phase).toBe('Ready')
    expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
    expect(mocks.coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(3) // existence, conflict, readiness
    expect(mocks.coreApi.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
  })

  it('Deployment 409 (already exists) triggers a replace via the conflict-retry helper', async () => {
    const mocks = makeMocks()
    mocks.appsApi.createNamespacedDeployment.mockRejectedValueOnce(alreadyExists())
    const drifted = liveDeployment()
    drifted.spec!.replicas = 9
    mocks.appsApi.readNamespacedDeployment
      .mockRejectedValueOnce(notFound())
      .mockResolvedValueOnce(drifted)
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcile(makeSfs())
    expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(3)
    expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
    expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalled()
  })

  it('preserves external annotations without replacing the desired Deployment spec', async () => {
    const mocks = makeMocks()
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
      metadata: {
        ...liveDeployment().metadata,
        resourceVersion: '7',
        annotations: { 'operator.example/keep': 'yes' },
      },
      spec: {
        replicas: 9,
        strategy: { type: 'RollingUpdate' },
        selector: { matchLabels: { app: 'stale-wfc' } },
        template: {
          metadata: {
            labels: { app: 'stale-wfc' },
            annotations: {
              'kubectl.kubernetes.io/restartedAt': '2026-08-22T12:00:00Z',
              'operator.example/pod-marker': 'keep',
            },
          },
          spec: {
            automountServiceAccountToken: true,
            imagePullSecrets: [{ name: 'stale-pull-secret' }],
            securityContext: { runAsUser: 2000 },
            initContainers: [{ name: 'stale-init', image: 'registry.example/stale-init:old' }],
            containers: [{ name: 'stale-wfc', image: 'registry.example/stale-wfc:old' }],
            volumes: [{ name: 'stale-volume', emptyDir: {} }],
          },
        },
      },
      status: { readyReplicas: 1, availableReplicas: 1 },
    })
    const reconciler = makeReconciler(mocks)

    await reconciler.reconcile(makeSfs())

    const desiredBody = buildDeployment(makeSfs(), factoryConfig)
    expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    const replaceBody = mocks.appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
    expect(replaceBody.metadata.annotations).toMatchObject({ 'operator.example/keep': 'yes' })
    expect(replaceBody.spec.template.metadata.annotations).toEqual({
      'kubectl.kubernetes.io/restartedAt': '2026-08-22T12:00:00Z',
      'operator.example/pod-marker': 'keep',
    })
    expect(replaceBody.spec.replicas).toBe(desiredBody.spec!.replicas)
    expect(replaceBody.spec.strategy).toEqual(desiredBody.spec!.strategy)
    expect(replaceBody.spec.selector).toEqual(desiredBody.spec!.selector)
    expect(replaceBody.spec.template.metadata.labels).toEqual(
      desiredBody.spec!.template.metadata!.labels
    )
    expect(replaceBody.spec.template.spec.containers).toEqual(
      desiredBody.spec!.template.spec!.containers
    )
    expect(replaceBody.spec.template.spec.initContainers).toEqual(
      desiredBody.spec!.template.spec!.initContainers
    )
    expect(replaceBody.spec.template.spec).toEqual(desiredBody.spec!.template.spec)
  })

  it('never creates a Job across repeated reconciles', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    await reconciler.reconcile(sfs)
    expect(mocks.batchApi.createNamespacedJob).not.toHaveBeenCalled()
  })
})

describe('SharedFileSystemReconciler — failure handling', () => {
  it('writes phase=Failed with a Reconciled=False condition when PVC create errors (non-409)', async () => {
    const mocks = makeMocks()
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(notFound())
    mocks.coreApi.createNamespacedPersistentVolumeClaim.mockRejectedValueOnce(
      err(500, 'cluster down')
    )
    const reconciler = makeReconciler(mocks)
    await expect(reconciler.reconcile(makeSfs())).rejects.toThrow('cluster down')
    const status = reconciler.getStatus(makeSfs())
    expect(status.phase).toBe('Failed')
    expect(status.conditions?.[0]).toMatchObject({
      type: 'Reconciled',
      status: 'False',
      reason: 'ReconcileError',
    })
    expect(mocks.customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
  })

  it('fails the reconcile on a hostile directory (validation in the wfc builder)', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    const sfs = makeSfs({ spec: { directories: ['../escape'] } })
    await expect(reconciler.reconcile(sfs)).rejects.toThrow(/SharedFileSystem director/i)
  })
})

describe('SharedFileSystemReconciler — reconcileDelete', () => {
  it('deletes Deployment, Service, and both NetworkPolicies but RETAINS PVC by default', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcileDelete('team-mission', 'mcp-host', { retainOnDelete: true })
    expect(mocks.networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
      expect.objectContaining({
        name: wfcIngressPolicyName({ name: 'team-mission', namespace: 'mcp-host' }),
      })
    )
    expect(mocks.networkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith(
      expect.objectContaining({
        name: wfcEgressPolicyName({ name: 'team-mission', namespace: 'mcp-host' }),
      })
    )
    expect(mocks.coreApi.deleteNamespacedService).toHaveBeenCalled()
    expect(mocks.appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
    expect(mocks.coreApi.deleteNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    // Also reaps any leftover legacy init Jobs on delete.
    expect(mocks.batchApi.listNamespacedJob).toHaveBeenCalled()
  })

  it('deletes the PVC when retainOnDelete=false', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcileDelete('team-mission', 'mcp-host', { retainOnDelete: false })
    expect(mocks.coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalledWith(
      expect.objectContaining({
        name: pvcName({ name: 'team-mission', namespace: 'mcp-host' }),
        namespace: 'mcp-host',
      })
    )
  })

  it('reaps legacy init Jobs BEFORE deleting the PVC (retainOnDelete=false)', async () => {
    // A still-running legacy init pod holding the RWO PVC would block reclaim
    // (pvc-protection finalizer) if the PVC were deleted first. Seed a matching
    // legacy Job so the actual delete is issued, and assert its order.
    const mocks = makeMocks()
    const sfs = makeSfs()
    mocks.batchApi.listNamespacedJob.mockResolvedValue({ items: [legacyJob(sfs)] })
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcileDelete('team-mission', 'mcp-host', { retainOnDelete: false })
    expect(mocks.batchApi.deleteNamespacedJob).toHaveBeenCalled()
    expect(mocks.coreApi.deleteNamespacedPersistentVolumeClaim).toHaveBeenCalled()
    const reapOrder = mocks.batchApi.deleteNamespacedJob.mock.invocationCallOrder[0]
    const pvcDeleteOrder =
      mocks.coreApi.deleteNamespacedPersistentVolumeClaim.mock.invocationCallOrder[0]
    expect(reapOrder).toBeLessThan(pvcDeleteOrder)
  })

  it('treats any 404 during teardown as success (resource already gone)', async () => {
    const mocks = makeMocks()
    mocks.coreApi.deleteNamespacedService.mockRejectedValueOnce(notFound())
    mocks.appsApi.deleteNamespacedDeployment.mockRejectedValueOnce(notFound())
    mocks.networkingApi.deleteNamespacedNetworkPolicy.mockRejectedValue(notFound())
    const reconciler = makeReconciler(mocks)
    await expect(
      reconciler.reconcileDelete('team-mission', 'mcp-host', { retainOnDelete: true })
    ).resolves.toBeUndefined()
  })

  it('clears in-memory status so a subsequent re-create starts fresh', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
    await reconciler.reconcileDelete(sfs.name, sfs.namespace, { retainOnDelete: true })
    expect(reconciler.getStatus(sfs).phase).toBe('Provisioning')
  })
})

describe('SharedFileSystemReconciler — fullReconcile', () => {
  it('reconciles every desired SFS and continues past errors on individual SFSes', async () => {
    const desired = [
      makeSfs({ name: 'sfs-a' }),
      makeSfs({ name: 'sfs-b' }),
      makeSfs({ name: 'sfs-c' }),
    ]
    const mocks = makeMocks(desired)
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockImplementation(async ({ name }) => {
      const sfs = desired.find(item => pvcName(item) === name)
      if (!sfs) throw new Error(`Missing PVC fixture for ${name}`)
      if (sfs.name === 'sfs-b') throw err(500, 'cluster down')
      return livePvc(sfs)
    })
    const reconciler = makeReconciler(mocks)
    await expect(reconciler.fullReconcile(desired)).resolves.toBeUndefined()
    expect(
      new Set(
        mocks.coreApi.readNamespacedPersistentVolumeClaim.mock.calls.map(([{ name }]) => name)
      )
    ).toEqual(new Set(desired.map(pvcName)))
    expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(reconciler.getStatus(desired[0]).phase).toBe('Ready')
    expect(reconciler.getStatus(desired[1]).phase).toBe('Failed')
    expect(reconciler.getStatus(desired[2]).phase).toBe('Ready')
  })
})

describe('SharedFileSystemReconciler — wfc env wiring (sanity)', () => {
  it('passes WSF_MOUNT_PATH=/workspace and SFS identity into the Deployment env', async () => {
    const mocks = makeMocks()
    mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
    const reconciler = makeReconciler(mocks)
    await reconciler.reconcile(makeSfs())
    const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body
    const env = dep.spec.template.spec.containers[0].env
    const byName = Object.fromEntries(env.map((e: { name: string; value?: string }) => [e.name, e]))
    expect(byName.WSF_MOUNT_PATH.value).toBe(WFC_MOUNT_PATH)
    expect(byName.WSF_SHARED_FILESYSTEM_NAME.value).toBe('team-mission')
    expect(byName.WSF_SHARED_FILESYSTEM_NAMESPACE.value).toBe('mcp-host')
  })
})

describe('SharedFileSystemReconciler — truthful status (#592)', () => {
  let mocks: ReturnType<typeof makeMocks>
  let reconciler: SharedFileSystemReconciler

  beforeEach(() => {
    mocks = makeMocks()
    reconciler = makeReconciler(mocks)
  })

  // PVC Pending under WaitForFirstConsumer while the wfc pod is still scheduling
  // is the NORMAL warm-up path — must report Initializing, never a false Degraded.
  it('reports Initializing (not Degraded) while the PVC is Pending and no pod is wedged', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(
      livePvc(makeSfs(), { phase: 'Pending' })
    )
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod not created yet
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
  })

  it('reports Degraded (PVCUnbound) when the wfc Pod is Unschedulable', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(
      livePvc(makeSfs(), { phase: 'Pending' })
    )
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({
      items: [
        {
          status: {
            conditions: [
              {
                type: 'PodScheduled',
                status: 'False',
                reason: 'Unschedulable',
                message: '0/2 nodes are available: 2 node(s) had volume node affinity conflict.',
              },
            ],
          },
        },
      ],
    })
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    const status = reconciler.getStatus(sfs)
    expect(status.phase).toBe('Degraded')
    expect(status.conditions?.[0]).toMatchObject({
      type: 'Reconciled',
      status: 'False',
      reason: 'PVCUnbound',
    })
  })

  // An Unschedulable that is NOT a volume/PVC problem (taint, insufficient CPU)
  // must NOT be mislabeled PVCUnbound, and must NOT carry the "check the PVC
  // accessModes" hint that would misdirect the operator. (Review fix #592.)
  it('reports Degraded (Unschedulable, NOT PVCUnbound) for a non-volume scheduling failure', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(
      livePvc(makeSfs(), { phase: 'Pending' })
    )
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({
      items: [
        {
          status: {
            conditions: [
              {
                type: 'PodScheduled',
                status: 'False',
                reason: 'Unschedulable',
                message:
                  '0/2 nodes are available: 2 Insufficient cpu, 2 node(s) had untolerated taint {node-role.kubernetes.io/control-plane: }.',
              },
            ],
          },
        },
      ],
    })
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    const status = reconciler.getStatus(sfs)
    expect(status.phase).toBe('Degraded')
    expect(status.conditions?.[0]?.reason).toBe('Unschedulable')
    expect(status.conditions?.[0]?.message ?? '').not.toMatch(/accessModes/i)
  })

  it('reports Degraded (WfcNotReady) when the wfc container is CrashLoopBackOff', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(
      livePvc(makeSfs(), { phase: 'Bound' })
    )
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({
      items: [
        {
          status: {
            containerStatuses: [
              {
                name: 'workspace-files-controller',
                state: { waiting: { reason: 'CrashLoopBackOff', message: 'back-off 5m0s' } },
              },
            ],
          },
        },
      ],
    })
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    const status = reconciler.getStatus(sfs)
    expect(status.phase).toBe('Degraded')
    expect(status.conditions?.[0]).toMatchObject({ reason: 'WfcNotReady' })
  })

  // The sfsResyncTimer re-runs reconcile; a transient Initializing must converge
  // to Ready once the PVC binds and the wfc pod serves — no manual intervention.
  it('auto-recovers to Ready on a later reconcile once the PVC binds and the wfc is ready', async () => {
    const sfs = makeSfs()
    mocks.coreApi.readNamespacedPersistentVolumeClaim
      .mockResolvedValueOnce(livePvc(makeSfs(), { phase: 'Pending' }))
      .mockResolvedValueOnce(livePvc(makeSfs(), { phase: 'Pending' }))
    mocks.appsApi.readNamespacedDeployment
      .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
      .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
    // Second reconcile falls through to the default happy-path mocks.
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
  })

  it('populates status.capacity from the bound PVC', async () => {
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).capacity).toBe('5Gi')
  })
})

describe('SharedFileSystemReconciler — status-write dirty check + lastTransitionTime (#592 oscillation)', () => {
  it('skips the no-op /status patch when re-reconciling a steady Ready SFS', async () => {
    const mocks = makeMocks()
    const reconciler = makeReconciler(mocks)
    const sfs = makeSfs()

    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
    const writesAfterFirst = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length
    expect(writesAfterFirst).toBe(1)

    // Re-reconciling an unchanged Ready SFS must NOT issue another /status patch
    // — a write would bump resourceVersion, fire a MODIFIED watch event, and
    // re-trigger reconcile, spinning a tight loop.
    await reconciler.reconcile(sfs)
    await reconciler.reconcile(sfs)
    expect(mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length).toBe(
      writesAfterFirst
    )
  })

  it('preserves lastTransitionTime across re-reconciles of an unchanged condition (real clock)', async () => {
    const mocks = makeMocks()
    let nowIdx = 0
    const times = [new Date('2026-04-30T12:00:00Z'), new Date('2026-04-30T12:05:00Z')]
    const reconciler = makeReconciler(mocks, () => times[nowIdx])
    const sfs = makeSfs()

    await reconciler.reconcile(sfs)
    const writesAfterFirst = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length
    expect(reconciler.getStatus(sfs).conditions?.[0]?.lastTransitionTime).toBe(
      '2026-04-30T12:00:00.000Z'
    )

    // Clock advances 5 minutes; the condition is still Ready (unchanged), so its
    // lastTransitionTime must be PRESERVED (K8s convention) — which is exactly
    // what keeps the serialized status identical so the dirty check can skip.
    nowIdx = 1
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).conditions?.[0]?.lastTransitionTime).toBe(
      '2026-04-30T12:00:00.000Z'
    )
    expect(mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length).toBe(
      writesAfterFirst
    )
  })

  it('writes again (and bumps lastTransitionTime) when the phase actually transitions', async () => {
    const mocks = makeMocks()
    let nowIdx = 0
    const times = [new Date('2026-04-30T12:00:00Z'), new Date('2026-04-30T12:05:00Z')]
    const reconciler = makeReconciler(mocks, () => times[nowIdx])
    const sfs = makeSfs()

    // First reconcile: PVC still Pending, no wedged pod → Initializing.
    mocks.coreApi.readNamespacedPersistentVolumeClaim
      .mockResolvedValueOnce(livePvc(makeSfs(), { phase: 'Pending' }))
      .mockResolvedValueOnce(livePvc(makeSfs(), { phase: 'Pending' }))
    mocks.appsApi.readNamespacedDeployment
      .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
      .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
    const writesAfterFirst = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length

    // Second reconcile (clock advanced): falls through to happy-path mocks → Ready.
    // The phase transitioned, so the status differs and MUST be written, with a
    // fresh lastTransitionTime reflecting the real transition.
    nowIdx = 1
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
    expect(mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length).toBeGreaterThan(
      writesAfterFirst
    )
    expect(reconciler.getStatus(sfs).conditions?.[0]?.lastTransitionTime).toBe(
      '2026-04-30T12:05:00.000Z'
    )
  })
})

describe('SharedFileSystemReconciler — mountability decoupled from wfc readiness (#592 mount resilience)', () => {
  let mocks: ReturnType<typeof makeMocks>
  let reconciler: SharedFileSystemReconciler

  beforeEach(() => {
    mocks = makeMocks()
    reconciler = makeReconciler(mocks)
  })

  it('isMountable is TRUE while Ready (PVC Bound + wfc serving)', async () => {
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Ready')
    expect(reconciler.isMountable(sfs)).toBe(true)
  })

  // The core resilience guarantee: a transient wfc HTTP outage (readyReplicas→0)
  // drops the SFS out of Ready, but the PVC stays Bound — so the RO mount MUST
  // remain mountable and never be ripped out of a running consumer pod.
  it('isMountable STAYS TRUE when the wfc is not serving but the PVC is Bound (Initializing)', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(livePvc())
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod restarting
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
    expect(reconciler.isMountable(sfs)).toBe(true)
  })

  it('isMountable STAYS TRUE on a Bound PVC even when the wfc container is CrashLoopBackOff (Degraded)', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(livePvc())
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({
      items: [
        {
          status: {
            containerStatuses: [
              {
                name: 'workspace-files-controller',
                state: { waiting: { reason: 'CrashLoopBackOff', message: 'back-off 5m0s' } },
              },
            ],
          },
        },
      ],
    })
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.getStatus(sfs).phase).toBe('Degraded')
    expect(reconciler.isMountable(sfs)).toBe(true)
  })

  it('isMountable is FALSE while the PVC is not Bound (Pending) — nothing to mount', async () => {
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue(
      livePvc(makeSfs(), { phase: 'Pending' })
    )
    mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
      liveDeployment(makeSfs(), { readyReplicas: 0 })
    )
    mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] })
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.isMountable(sfs)).toBe(false)
  })

  it('isMountable defaults to FALSE for an unknown SFS', () => {
    expect(reconciler.isMountable(makeSfs({ name: 'never-seen' }))).toBe(false)
  })

  it('clears mountability on delete', async () => {
    const sfs = makeSfs()
    await reconciler.reconcile(sfs)
    expect(reconciler.isMountable(sfs)).toBe(true)
    await reconciler.reconcileDelete(sfs.name, sfs.namespace, { retainOnDelete: true })
    expect(reconciler.isMountable(sfs)).toBe(false)
  })
})

// Exercise the actual resource adapters without readiness reads masking their
// physical GET counts. Integration status and recovery remain covered above.
describe.each(['Deployment', 'Service', 'PersistentVolumeClaim'] as const)(
  'SFS %s read-first contract',
  kind => {
    beforeEach(() => {
      createsTotal.reset()
      existenceReadsTotal.reset()
    })
    function setup() {
      const mocks = makeMocks()
      const reconciler = makeReconciler(mocks) as unknown as {
        ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>
        ensureService(sfs: SharedFileSystemCRD): Promise<string>
        ensurePvc(sfs: SharedFileSystemCRD): Promise<string>
      }
      const sfs = makeSfs()
      const selected = {
        Deployment: {
          name: wfcDeploymentName(sfs),
          read: mocks.appsApi.readNamespacedDeployment,
          create: mocks.appsApi.createNamespacedDeployment,
          replace: mocks.appsApi.replaceNamespacedDeployment,
          run: () => reconciler.ensureDeployment(sfs),
        },
        Service: {
          name: wfcServiceName(sfs),
          read: mocks.coreApi.readNamespacedService,
          create: mocks.coreApi.createNamespacedService,
          replace: mocks.coreApi.replaceNamespacedService,
          run: () => reconciler.ensureService(sfs),
        },
        PersistentVolumeClaim: {
          name: pvcName(sfs),
          read: mocks.coreApi.readNamespacedPersistentVolumeClaim,
          create: mocks.coreApi.createNamespacedPersistentVolumeClaim,
          replace: mocks.coreApi.replaceNamespacedPersistentVolumeClaim,
          run: () => reconciler.ensurePvc(sfs),
        },
      }[kind]
      const { name, read, create, replace, run } = selected
      return { mocks, sfs, name, read, create, replace, run }
    }
    async function count(metric: typeof createsTotal, outcome: string) {
      return (
        (await metric.get()).values.find(
          row => row.labels.kind === kind && row.labels.outcome === outcome
        )?.value ?? 0
      )
    }

    it('returns the existing name with one GET and no POST or PUT', async () => {
      const h = setup()
      await expect(h.run()).resolves.toBe(h.name)
      expect(h.read).toHaveBeenCalledExactlyOnceWith({ name: h.name, namespace: 'mcp-host' })
      expect(h.create).not.toHaveBeenCalled()
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'skipped')).toBe(1)
      expect(await count(existenceReadsTotal, 'found')).toBe(1)
    })

    it('creates only after GET404 and returns the created name', async () => {
      const h = setup()
      h.read.mockRejectedValueOnce(notFound())
      await expect(h.run()).resolves.toBe(h.name)
      expect(h.read).toHaveBeenCalledTimes(1)
      expect(h.create).toHaveBeenCalledExactlyOnceWith({
        namespace: 'mcp-host',
        body:
          kind === 'Deployment'
            ? buildDeployment(h.sfs, factoryConfig)
            : kind === 'Service'
              ? buildService(h.sfs, factoryConfig)
              : buildPvc(h.sfs, factoryConfig),
      })
      expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'created')).toBe(1)
      expect(await count(createsTotal, 'skipped')).toBe(0)
    })

    it('consumes a fresh GET after POST409 rather than reusing absence', async () => {
      const h = setup()
      h.read.mockRejectedValueOnce(notFound())
      h.create.mockRejectedValueOnce(alreadyExists())
      await expect(h.run()).resolves.toBe(h.name)
      expect(h.read).toHaveBeenCalledTimes(2)
      expect(h.create).toHaveBeenCalledTimes(1)
      expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])
      expect(h.create.mock.invocationCallOrder[0]).toBeLessThan(h.read.mock.invocationCallOrder[1])
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'conflict')).toBe(1)
      expect(await count(createsTotal, 'skipped')).toBe(0)
    })

    it.each([403, 500])('propagates GET%s without any writes or skip', async code => {
      const h = setup()
      const failure = err(code)
      h.read.mockRejectedValueOnce(failure)
      await expect(h.run()).rejects.toBe(failure)
      expect(h.read).toHaveBeenCalledTimes(1)
      expect(h.create).not.toHaveBeenCalled()
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'skipped')).toBe(0)
      expect(await count(existenceReadsTotal, 'error')).toBe(1)
    })

    it('preserves benign disappearance after POST409 without another create or skip', async () => {
      const h = setup()
      h.read.mockRejectedValueOnce(notFound()).mockRejectedValueOnce(notFound())
      h.create.mockRejectedValueOnce(alreadyExists())
      await expect(h.run()).resolves.toBe(h.name)
      expect(h.read).toHaveBeenCalledTimes(2)
      expect(h.create).toHaveBeenCalledTimes(1)
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'skipped')).toBe(0)
    })

    it('propagates fresh GET403 after POST409', async () => {
      const h = setup()
      const failure = err(403)
      h.read.mockRejectedValueOnce(notFound()).mockRejectedValueOnce(failure)
      h.create.mockRejectedValueOnce(alreadyExists())
      await expect(h.run()).rejects.toBe(failure)
      expect(h.read).toHaveBeenCalledTimes(2)
      expect(h.create).toHaveBeenCalledTimes(1)
      expect(h.replace).not.toHaveBeenCalled()
      expect(await count(createsTotal, 'skipped')).toBe(0)
    })
  }
)

describe('SFS preserve and retry contracts', () => {
  it('keeps an existing PVC unchanged even when desired size or access modes differ', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    const pvc = livePvc(sfs)
    pvc.spec = {
      ...pvc.spec,
      accessModes: ['ReadOnlyMany'],
      resources: { requests: { storage: '9Gi' } },
    }
    mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce(pvc)
    const reconciler = makeReconciler(mocks) as unknown as {
      ensurePvc(sfs: SharedFileSystemCRD): Promise<string>
    }
    await expect(reconciler.ensurePvc(sfs)).resolves.toBe(pvcName(sfs))
    expect(mocks.coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
    expect(mocks.coreApi.createNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(mocks.coreApi.replaceNamespacedPersistentVolumeClaim).not.toHaveBeenCalled()
    expect(pvc.spec.accessModes).toEqual(['ReadOnlyMany'])
    expect(pvc.spec.resources?.requests?.storage).toBe('9Gi')
  })

  it('keeps an existing Service unchanged even when its fields differ from desired', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    const service = buildService(sfs, factoryConfig)
    service.spec = { ...service.spec, clusterIP: '10.96.1.45', ports: [{ port: 9090 }] }
    mocks.coreApi.readNamespacedService.mockResolvedValueOnce(service)
    const reconciler = makeReconciler(mocks) as unknown as {
      ensureService(sfs: SharedFileSystemCRD): Promise<string>
    }
    await expect(reconciler.ensureService(sfs)).resolves.toBe(wfcServiceName(sfs))
    expect(mocks.coreApi.readNamespacedService).toHaveBeenCalledTimes(1)
    expect(mocks.coreApi.createNamespacedService).not.toHaveBeenCalled()
    expect(mocks.coreApi.replaceNamespacedService).not.toHaveBeenCalled()
    expect(service.spec.clusterIP).toBe('10.96.1.45')
    expect(service.spec.ports).toEqual([{ port: 9090 }])
  })

  it('reuses the first Deployment snapshot once and rereads its version after PUT409', async () => {
    const mocks = makeMocks()
    const sfs = makeSfs()
    let version = 0
    mocks.appsApi.readNamespacedDeployment.mockImplementation(async () => {
      const live = liveDeployment(sfs)
      live.metadata!.resourceVersion = String(++version)
      live.spec!.replicas = 9
      return live
    })
    mocks.appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(alreadyExists())
    const reconciler = makeReconciler(mocks) as unknown as {
      ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>
    }
    await expect(reconciler.ensureDeployment(sfs)).resolves.toBe(wfcDeploymentName(sfs))
    expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
    expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
    expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
    expect(
      mocks.appsApi.replaceNamespacedDeployment.mock.calls.map(
        ([{ body }]) => body.metadata.resourceVersion
      )
    ).toEqual(['1', '2'])
  })
})