Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
48.2 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import { governedTraceAdministrativeIntentReconcileFailedTotal } from '../src/observability/metrics.js'
import {
  K8sConflictError,
  K8sNotFoundError,
  ResourceService,
  setAdministrativeOperationService,
} from '../src/services/resourceService.js'
import { runWithAdministrativeRequestContext } from '../src/services/tracing/adminOperationContext.js'

function makeNotFoundError(): Error & { statusCode: number } {
  const err = new Error('not found') as Error & { statusCode: number }
  err.statusCode = 404
  return err
}

function makeConflictError(): Error & { statusCode: number } {
  const err = new Error('object has been modified') as Error & { statusCode: number }
  err.statusCode = 409
  return err
}

describe('ResourceService.getResource', () => {
  it('does not fall back cluster-wide when an explicit namespace probe misses', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn(async ({ namespace }: { namespace: string }) => {
        if (namespace === 'mcp-server') throw makeNotFoundError()
        return { metadata: { name: 'sandbox-only', namespace } }
      }),
      listNamespacedCustomObject: vi.fn(async ({ namespace }: { namespace: string }) => {
        if (namespace === 'sandbox-recipes') {
          return { items: [{ metadata: { name: 'sandbox-only', namespace } }] }
        }
        return { items: [] }
      }),
    }

    const service = new ResourceService(customApi as never, 'control-plane', {
      workflowrecipes: ['mcp-server', 'sandbox-recipes'],
    })

    await expect(
      service.getResource('workflowrecipes', 'sandbox-only', 'mcp-server')
    ).rejects.toBeInstanceOf(K8sNotFoundError)
    expect(customApi.listNamespacedCustomObject).not.toHaveBeenCalled()
  })

  it('still falls back cluster-wide for implicit primary-namespace lookups', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn(async () => {
        throw makeNotFoundError()
      }),
      listNamespacedCustomObject: vi.fn(async ({ namespace }: { namespace: string }) => {
        if (namespace === 'sandbox-recipes') {
          return { items: [{ metadata: { name: 'sandbox-only', namespace } }] }
        }
        return { items: [] }
      }),
    }

    const service = new ResourceService(customApi as never, 'control-plane', {
      workflowrecipes: ['mcp-server', 'sandbox-recipes'],
    })

    const found = (await service.getResource('workflowrecipes', 'sandbox-only')) as {
      metadata?: { namespace?: string }
    }
    expect(found.metadata?.namespace).toBe('sandbox-recipes')
    expect(customApi.listNamespacedCustomObject).toHaveBeenCalled()
  })

  it('propagates non-404 errors for explicit namespace probes', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn(async () => {
        const err = new Error('forbidden') as Error & { statusCode: number }
        err.statusCode = 403
        throw err
      }),
      listNamespacedCustomObject: vi.fn(),
    }

    const service = new ResourceService(customApi as never, 'control-plane', {
      workflowrecipes: ['mcp-server', 'sandbox-recipes'],
    })

    await expect(
      service.getResource('workflowrecipes', 'sandbox-only', 'mcp-server')
    ).rejects.toMatchObject({ statusCode: 403 })
    expect(customApi.listNamespacedCustomObject).not.toHaveBeenCalled()
  })
})

describe('ResourceService.updateResource', () => {
  it('forwards UID together with resourceVersion for an identity-bound replacement', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'cc-a',
          namespace: 'channels',
          uid: 'uid-original',
          resourceVersion: '10',
          ownerReferences: [
            {
              apiVersion: 'clerum.io/v1alpha1',
              kind: 'WorkflowRecipe',
              name: 'parent',
              uid: 'uid-parent',
            },
          ],
          finalizers: ['clerum.io/workload-cleanup'],
        },
        spec: { enabled: true },
      }),
      replaceNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'cc-a',
          namespace: 'channels',
          uid: 'uid-original',
          resourceVersion: '11',
        },
      }),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', {
      communicationchannels: 'channels',
    })

    await service.updateResource(
      'communicationchannels',
      'cc-a',
      {
        metadata: { uid: 'uid-original', resourceVersion: '10' },
        spec: { enabled: false },
      },
      'channels'
    )

    const replaceArgs = customApi.replaceNamespacedCustomObject.mock.calls[0][0]
    expect(replaceArgs.body.metadata.uid).toBe('uid-original')
    expect(replaceArgs.body.metadata.resourceVersion).toBe('10')
    expect(replaceArgs.body.metadata.ownerReferences).toEqual([
      {
        apiVersion: 'clerum.io/v1alpha1',
        kind: 'WorkflowRecipe',
        name: 'parent',
        uid: 'uid-parent',
      },
    ])
    expect(replaceArgs.body.metadata.finalizers).toEqual(['clerum.io/workload-cleanup'])
  })

  it('fails closed before replacement when the identity precondition names a replacement object', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'cc-a',
          namespace: 'channels',
          uid: 'uid-replacement',
          resourceVersion: '10',
        },
        spec: { enabled: true },
      }),
      replaceNamespacedCustomObject: vi.fn(),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', {
      communicationchannels: 'channels',
    })

    await expect(
      service.updateResource(
        'communicationchannels',
        'cc-a',
        {
          metadata: { uid: 'uid-original', resourceVersion: '10' },
          spec: { enabled: false },
        },
        'channels'
      )
    ).rejects.toBeInstanceOf(K8sConflictError)
    expect(customApi.replaceNamespacedCustomObject).not.toHaveBeenCalled()
  })

  it('refetches and retries once when Kubernetes reports a resourceVersion conflict', async () => {
    const customApi = {
      getNamespacedCustomObject: vi
        .fn()
        .mockResolvedValueOnce({
          metadata: { name: 'mcp-a', namespace: 'mcp-server', resourceVersion: '10' },
          spec: { enabled: true },
        })
        .mockResolvedValueOnce({
          metadata: { name: 'mcp-a', namespace: 'mcp-server', resourceVersion: '11' },
          spec: { enabled: true },
        }),
      replaceNamespacedCustomObject: vi
        .fn()
        .mockRejectedValueOnce(makeConflictError())
        .mockResolvedValueOnce({
          metadata: { name: 'mcp-a', namespace: 'mcp-server', resourceVersion: '12' },
          spec: { enabled: true, egressBindings: [{ egressClass: 'public-web' }] },
        }),
      listNamespacedCustomObject: vi.fn(),
    }

    const service = new ResourceService(customApi as never, 'control-plane', {
      mcpservers: 'mcp-server',
    })

    const updated = (await service.updateResource(
      'mcpservers',
      'mcp-a',
      { spec: { enabled: true, egressBindings: [{ egressClass: 'public-web' }] } },
      'mcp-server'
    )) as { metadata?: { resourceVersion?: string } }

    expect(updated.metadata?.resourceVersion).toBe('12')
    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(2)
    const firstReplace = customApi.replaceNamespacedCustomObject.mock.calls[0][0]
    const secondReplace = customApi.replaceNamespacedCustomObject.mock.calls[1][0]
    expect(firstReplace.body.metadata.resourceVersion).toBe('10')
    expect(secondReplace.body.metadata.resourceVersion).toBe('11')
  })
})

describe('ResourceService.deleteResource', () => {
  it('forwards UID and resourceVersion preconditions to Kubernetes', async () => {
    const customApi = {
      deleteNamespacedCustomObject: vi.fn(async () => ({ deleted: true })),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', {
      mcpservers: 'mcp-server',
    })

    await service.deleteResource('mcpservers', 'mcp-a', 'mcp-server', {
      uid: 'uid-a',
      resourceVersion: '7',
    })

    expect(customApi.deleteNamespacedCustomObject).toHaveBeenCalledWith({
      group: 'clerum.io',
      version: 'v1alpha1',
      namespace: 'mcp-server',
      plural: 'mcpservers',
      name: 'mcp-a',
      body: { preconditions: { uid: 'uid-a', resourceVersion: '7' } },
    })
  })
})

describe('ResourceService.updateResource — AP-6 reader-version precondition', () => {
  function makeCustomApi(
    overrides: {
      currentAnnotations?: Record<string, string>
      currentUid?: string
      replace?: ReturnType<typeof vi.fn>
    } = {}
  ) {
    return {
      getNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'host-a',
          namespace: 'mcp-host',
          resourceVersion: '11',
          ...(overrides.currentAnnotations && { annotations: overrides.currentAnnotations }),
          ...(overrides.currentUid && { uid: overrides.currentUid }),
        },
        spec: { contextRef: 'live' },
      }),
      replaceNamespacedCustomObject:
        overrides.replace ??
        vi.fn().mockResolvedValue({
          metadata: { name: 'host-a', namespace: 'mcp-host', resourceVersion: '12' },
        }),
      listNamespacedCustomObject: vi.fn(),
    }
  }

  it('uses the caller-provided identity and resourceVersion in the replacement', async () => {
    const customApi = makeCustomApi({ currentUid: 'uid-reader' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource(
      'hosts',
      'host-a',
      { metadata: { uid: 'uid-reader', resourceVersion: '10' }, spec: { contextRef: 'edited' } },
      'mcp-host'
    )

    expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(1)
    const replaceArgs = customApi.replaceNamespacedCustomObject.mock.calls[0][0]
    // The precondition is the READER's version ('10'), not the fresher
    // server version ('11') harvested by the internal read.
    expect(replaceArgs.body.metadata.resourceVersion).toBe('10')
    expect(replaceArgs.body.metadata.uid).toBe('uid-reader')
  })

  it('surfaces a 409 as K8sConflictError WITHOUT retrying when the reader version is stale', async () => {
    const replace = vi.fn().mockRejectedValue(makeConflictError())
    const customApi = makeCustomApi({ replace, currentUid: 'uid-reader' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await expect(
      service.updateResource(
        'hosts',
        'host-a',
        {
          metadata: { uid: 'uid-reader', resourceVersion: '10' },
          spec: { contextRef: 'stale-edit' },
        },
        'mcp-host'
      )
    ).rejects.toBeInstanceOf(K8sConflictError)

    // No retry loop: exactly one read + one replace attempt. Retrying would
    // re-apply the same stale payload over the concurrent write.
    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
    expect(replace).toHaveBeenCalledTimes(1)
    expect(replace.mock.calls[0][0].body.metadata).toMatchObject({
      uid: 'uid-reader',
      resourceVersion: '10',
    })
  })

  it('retries a UID-only legacy write after a 409 while the refetched object keeps that UID', async () => {
    const customApi = {
      getNamespacedCustomObject: vi
        .fn()
        .mockResolvedValueOnce({
          metadata: { uid: 'uid-original', resourceVersion: '1' },
          spec: {},
        })
        .mockResolvedValueOnce({
          metadata: { uid: 'uid-original', resourceVersion: '2' },
          spec: {},
        }),
      replaceNamespacedCustomObject: vi
        .fn()
        .mockRejectedValueOnce(makeConflictError())
        .mockResolvedValueOnce({ metadata: { uid: 'uid-original', resourceVersion: '3' } }),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource(
      'hosts',
      'host-a',
      { metadata: { uid: 'uid-original' }, spec: { contextRef: 'edited' } },
      'mcp-host'
    )

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    const attempts = customApi.replaceNamespacedCustomObject.mock.calls.map(
      call => call[0].body.metadata
    )
    expect(attempts).toEqual([
      expect.objectContaining({ uid: 'uid-original', resourceVersion: '1' }),
      expect.objectContaining({ uid: 'uid-original', resourceVersion: '2' }),
    ])
  })

  it('does not write a same-name replacement after a UID-only retry observes a different UID', async () => {
    const replace = vi.fn().mockRejectedValueOnce(makeConflictError())
    const customApi = {
      getNamespacedCustomObject: vi
        .fn()
        .mockResolvedValueOnce({
          metadata: { uid: 'uid-original', resourceVersion: '1' },
          spec: {},
        })
        .mockResolvedValueOnce({
          metadata: { uid: 'uid-replacement', resourceVersion: '2' },
          spec: {},
        }),
      replaceNamespacedCustomObject: replace,
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await expect(
      service.updateResource(
        'hosts',
        'host-a',
        { metadata: { uid: 'uid-original' }, spec: { contextRef: 'edited' } },
        'mcp-host'
      )
    ).rejects.toBeInstanceOf(K8sConflictError)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    expect(replace).toHaveBeenCalledTimes(1)
  })

  it('AP-6 + concurrent wake bump: a stale reader version is rejected 409 while the newer clerum.io/wake-requested on the server is left untouched (no retry re-applies the stale spec)', async () => {
    const WAKE = 'clerum.io/wake-requested'
    // The server object moved on AFTER the admin read it: resourceVersion is now
    // '12' (reader saw '10') AND a concurrent hostWakeService projection bumped
    // clerum.io/wake-requested to '8'. mergeAnnotationsForReplace reads CURRENT
    // annotations at commit, so it WOULD carry the bump forward — but the stale
    // reader-version precondition makes K8s reject the whole replace first.
    const getNamespacedCustomObject = vi.fn().mockResolvedValue({
      metadata: {
        name: 'host-a',
        namespace: 'mcp-host',
        resourceVersion: '12',
        annotations: { [WAKE]: '8', team: 'blue' },
      },
      spec: { contextRef: 'live-with-wake-8' },
    })
    // K8s enforces optimistic concurrency: the replace carrying the reader's
    // stale precondition ('10') is rejected 409 and never mutates the object.
    const replaceNamespacedCustomObject = vi.fn().mockRejectedValue(makeConflictError())
    const customApi = {
      getNamespacedCustomObject,
      replaceNamespacedCustomObject,
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    // Property 1: the 409 is surfaced as K8sConflictError (not retried away).
    await expect(
      service.updateResource(
        'hosts',
        'host-a',
        {
          metadata: { resourceVersion: '10', annotations: { notes: 'stale-admin-edit' } },
          spec: { contextRef: 'stale-admin-spec' },
        },
        'mcp-host'
      )
    ).rejects.toBeInstanceOf(K8sConflictError)

    // No retry: exactly one read + one replace attempt. Re-reading only to
    // re-apply the same stale payload is the lost-update bug AP-6 prevents.
    expect(getNamespacedCustomObject).toHaveBeenCalledTimes(1)
    expect(replaceNamespacedCustomObject).toHaveBeenCalledTimes(1)

    // Property 2: the wake annotation on the server is UNTOUCHED. The single
    // replace attempt that fired carried the bumped clerum.io/wake-requested='8'
    // (mergeAnnotationsForReplace re-added it from the fresh read), so even the
    // rejected write would not have erased it — and since the write was rejected,
    // the server object keeps its concurrently-bumped wake generation intact.
    const attemptedAnnotations = replaceNamespacedCustomObject.mock.calls[0][0].body.metadata
      .annotations as Record<string, string>
    expect(attemptedAnnotations[WAKE]).toBe('8')
    // The stale caller's own key is present per replace semantics, but it never
    // reached the server because the precondition check rejected the write.
    expect(attemptedAnnotations.notes).toBe('stale-admin-edit')
  })

  it('pins the legacy path when no reader version is provided: server-version precondition + up to 3 attempts', async () => {
    const customApi = {
      getNamespacedCustomObject: vi
        .fn()
        .mockResolvedValueOnce({ metadata: { resourceVersion: '1' }, spec: {} })
        .mockResolvedValueOnce({ metadata: { resourceVersion: '2' }, spec: {} })
        .mockResolvedValueOnce({ metadata: { resourceVersion: '3' }, spec: {} }),
      replaceNamespacedCustomObject: vi
        .fn()
        .mockRejectedValueOnce(makeConflictError())
        .mockRejectedValueOnce(makeConflictError())
        .mockResolvedValueOnce({ metadata: { resourceVersion: '4' } }),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource('hosts', 'host-a', { spec: { contextRef: 'x' } }, 'mcp-host')

    // Each retry re-reads and uses the FRESH server version — the legacy
    // last-write-wins compat contract for RV-absent callers.
    expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledTimes(3)
    const attempts = customApi.replaceNamespacedCustomObject.mock.calls.map(
      call => call[0].body.metadata.resourceVersion
    )
    expect(attempts).toEqual(['1', '2', '3'])
  })
})

describe('ResourceService annotation merge — platform keys survive admin writes', () => {
  const WAKE = 'clerum.io/wake-requested'

  function makeCustomApi(currentAnnotations: Record<string, string>) {
    return {
      getNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'host-a',
          namespace: 'mcp-host',
          resourceVersion: '5',
          annotations: currentAnnotations,
        },
        spec: { contextRef: 'live' },
      }),
      replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
      listNamespacedCustomObject: vi.fn(),
    }
  }

  function replacedAnnotations(customApi: {
    replaceNamespacedCustomObject: ReturnType<typeof vi.fn>
  }): Record<string, string> | undefined {
    return customApi.replaceNamespacedCustomObject.mock.calls[0][0].body.metadata.annotations
  }

  it('updateResource: an unrelated annotations map does NOT erase clerum.io/wake-requested', async () => {
    const customApi = makeCustomApi({ [WAKE]: '7', team: 'blue' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource(
      'hosts',
      'host-a',
      { metadata: { annotations: { notes: 'hello' } }, spec: {} },
      'mcp-host'
    )

    // The platform projection survives; caller-owned keys keep replace
    // semantics (omitting `team` clears it).
    expect(replacedAnnotations(customApi)).toEqual({ notes: 'hello', [WAKE]: '7' })
  })

  it('updateResource: explicitly setting the exact platform key lets the caller win', async () => {
    const customApi = makeCustomApi({ [WAKE]: '7' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource(
      'hosts',
      'host-a',
      { metadata: { annotations: { [WAKE]: '9' } }, spec: {} },
      'mcp-host'
    )

    expect(replacedAnnotations(customApi)).toEqual({ [WAKE]: '9' })
  })

  it('updateResource: explicitly clearing an own key still works (map replaced per caller intent)', async () => {
    const customApi = makeCustomApi({ [WAKE]: '7', team: 'blue', notes: 'old' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource(
      'hosts',
      'host-a',
      { metadata: { annotations: { notes: 'new' } }, spec: {} },
      'mcp-host'
    )

    expect(replacedAnnotations(customApi)).toEqual({ notes: 'new', [WAKE]: '7' })
  })

  it('updateResource: a body without annotations preserves the whole current map (legacy)', async () => {
    const customApi = makeCustomApi({ [WAKE]: '7', team: 'blue' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.updateResource('hosts', 'host-a', { spec: {} }, 'mcp-host')

    expect(replacedAnnotations(customApi)).toEqual({ [WAKE]: '7', team: 'blue' })
  })

  it('mutateResource: a mutation carrying an unrelated annotations map does NOT erase platform keys', async () => {
    const customApi = makeCustomApi({ [WAKE]: '7' })
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    await service.mutateResource(
      'hosts',
      'host-a',
      () => ({ metadata: { annotations: { notes: 'x' } }, spec: {} }),
      'mcp-host'
    )

    expect(replacedAnnotations(customApi)).toEqual({ notes: 'x', [WAKE]: '7' })
  })
})

describe('ResourceService.mutateResource', () => {
  it('preserves lifecycle metadata from the live object during a full replace', async () => {
    const customApi = {
      getNamespacedCustomObject: vi.fn().mockResolvedValue({
        metadata: {
          name: 'recipe-a',
          namespace: 'sandbox-recipes',
          resourceVersion: '21',
          ownerReferences: [
            { apiVersion: 'v1', kind: 'WorkflowRecipe', name: 'parent', uid: 'uid-parent' },
          ],
          finalizers: ['clerum.io/workload-cleanup'],
        },
        spec: { enabled: true },
      }),
      replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', {
      workflowrecipes: 'sandbox-recipes',
    })

    await service.mutateResource(
      'workflowrecipes',
      'recipe-a',
      current => ({ spec: { ...current.spec, enabled: false } }),
      'sandbox-recipes'
    )

    const metadata = customApi.replaceNamespacedCustomObject.mock.calls[0][0].body.metadata
    expect(metadata.ownerReferences).toEqual([
      { apiVersion: 'v1', kind: 'WorkflowRecipe', name: 'parent', uid: 'uid-parent' },
    ])
    expect(metadata.finalizers).toEqual(['clerum.io/workload-cleanup'])
  })

  it('recomputes the replacement body from the refetched resource after a conflict', async () => {
    const customApi = {
      getNamespacedCustomObject: vi
        .fn()
        .mockResolvedValueOnce({
          metadata: { name: 'cc-a', namespace: 'channels', resourceVersion: '10' },
          spec: { telegram: [{ channelId: 'seed-chat', chatType: 'private' }] },
        })
        .mockResolvedValueOnce({
          metadata: { name: 'cc-a', namespace: 'channels', resourceVersion: '11' },
          spec: {
            telegram: [
              { channelId: 'seed-chat', chatType: 'private' },
              { channelId: 'concurrent-group', chatType: 'group' },
            ],
          },
        }),
      replaceNamespacedCustomObject: vi
        .fn()
        .mockRejectedValueOnce(makeConflictError())
        .mockResolvedValueOnce({
          metadata: { name: 'cc-a', namespace: 'channels', resourceVersion: '12' },
        }),
      listNamespacedCustomObject: vi.fn(),
    }
    const service = new ResourceService(customApi as never, 'control-plane', {
      communicationchannels: 'channels',
    })

    await service.mutateResource(
      'communicationchannels',
      'cc-a',
      current => {
        const telegram = Array.isArray(current.spec?.telegram) ? current.spec.telegram : []
        return {
          spec: {
            ...current.spec,
            telegram: [...telegram, { channelId: '777', chatType: 'private' }],
          },
        }
      },
      'channels'
    )

    const secondReplace = customApi.replaceNamespacedCustomObject.mock.calls[1][0]
    expect(secondReplace.body.metadata.resourceVersion).toBe('11')
    expect(secondReplace.body.spec.telegram).toEqual([
      { channelId: 'seed-chat', chatType: 'private' },
      { channelId: 'concurrent-group', chatType: 'group' },
      { channelId: '777', chatType: 'private' },
    ])
  })
})

describe('ResourceService.patchAnnotationMonotonic', () => {
  const NS = 'mcp-host'
  const KEY = 'clerum.io/wake-requested'

  function makeService(overrides: {
    get: (args: { name: string }) => unknown
    patch?: (...args: unknown[]) => unknown
  }) {
    const patch = overrides.patch ?? vi.fn(async () => ({ metadata: { resourceVersion: '2' } }))
    const customApi = {
      getNamespacedCustomObject: vi.fn(async ({ name }: { name: string }) =>
        overrides.get({ name })
      ),
      patchNamespacedCustomObject: patch,
      listNamespacedCustomObject: vi.fn(async () => ({ items: [] })),
    }
    const service = new ResourceService(customApi as never, NS, { hosts: NS })
    return { service, customApi, patch }
  }

  it('does NOT regress the projection when the incoming generation is lower than the projected one', async () => {
    // The Host already projects generation 6. A slower generation-5 wake must
    // NOT overwrite it (that is exactly the M1 ordering regression). The read
    // happens, but no patch is issued.
    const { service, customApi } = makeService({
      get: () => ({
        metadata: { resourceVersion: '10', annotations: { [KEY]: '6' } },
      }),
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 5, NS)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(1)
    expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
  })

  it('does NOT patch when the incoming generation equals the projected one', async () => {
    const { service, customApi } = makeService({
      get: () => ({
        metadata: { resourceVersion: '10', annotations: { [KEY]: '6' } },
      }),
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 6, NS)

    expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalled()
  })

  it('projects a strictly higher generation with a resourceVersion precondition', async () => {
    const { service, customApi } = makeService({
      get: () => ({
        metadata: { resourceVersion: '10', annotations: { [KEY]: '6' } },
      }),
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 7, NS)

    expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledTimes(1)
    const patchArgs = customApi.patchNamespacedCustomObject.mock.calls[0][0] as {
      body: { metadata: { resourceVersion?: string; annotations: Record<string, string> } }
    }
    expect(patchArgs.body.metadata.resourceVersion).toBe('10')
    expect(patchArgs.body.metadata.annotations[KEY]).toBe('7')
  })

  it('projects when no annotation exists yet (null projected value)', async () => {
    const { service, customApi } = makeService({
      get: () => ({ metadata: { resourceVersion: '3' } }),
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 1, NS)

    expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledTimes(1)
  })

  it('retries on 409 by re-reading, and then respects max-semantics (no regression on the winning value)', async () => {
    // First read shows generation 6 (rv=10); our patch conflicts (409) because
    // a concurrent higher-generation writer landed generation 8 (rv=11). On
    // re-read we observe 8 >= our 7, so we STOP without regressing.
    let reads = 0
    const patch = vi.fn(async () => {
      const err = new Error('conflict') as Error & { statusCode: number }
      err.statusCode = 409
      throw err
    })
    const { service, customApi } = makeService({
      get: () => {
        reads += 1
        return reads === 1
          ? { metadata: { resourceVersion: '10', annotations: { [KEY]: '6' } } }
          : { metadata: { resourceVersion: '11', annotations: { [KEY]: '8' } } }
      },
      patch,
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 7, NS)

    expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
    // One patch attempt (the 409), then re-read shows a higher value so no
    // further patch is attempted — the projection is never regressed.
    expect(patch).toHaveBeenCalledTimes(1)
  })

  it('treats a non-numeric existing annotation as no-known-value and self-heals to the DB generation', async () => {
    const { service, customApi } = makeService({
      get: () => ({
        metadata: { resourceVersion: '4', annotations: { [KEY]: 'garbage' } },
      }),
    })

    await service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 2, NS)

    expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledTimes(1)
    const patchArgs = customApi.patchNamespacedCustomObject.mock.calls[0][0] as {
      body: { metadata: { annotations: Record<string, string> } }
    }
    expect(patchArgs.body.metadata.annotations[KEY]).toBe('2')
  })

  it('fails loud when the resource has no resourceVersion (cannot enforce the precondition)', async () => {
    const { service } = makeService({
      get: () => ({ metadata: { annotations: {} } }),
    })

    await expect(service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 1, NS)).rejects.toThrow(
      /resourceVersion/
    )
  })

  it('maps a 404 on patch to K8sNotFoundError', async () => {
    const patch = vi.fn(async () => {
      const err = new Error('not found') as Error & { statusCode: number }
      err.statusCode = 404
      throw err
    })
    const { service } = makeService({
      get: () => ({ metadata: { resourceVersion: '1' } }),
      patch,
    })

    await expect(
      service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 1, NS)
    ).rejects.toBeInstanceOf(K8sNotFoundError)
  })

  it('rejects a non-integer generation loudly', async () => {
    const { service } = makeService({ get: () => ({ metadata: { resourceVersion: '1' } }) })

    await expect(
      service.patchAnnotationMonotonic('hosts', 'chatllm', KEY, 1.5, NS)
    ).rejects.toThrow(/non-negative integer/)
  })
})

describe('ResourceService Host administrative intent', () => {
  it('strips caller authority and projects the control-api operation id before create', async () => {
    const persistHostIntent = vi.fn().mockResolvedValue({
      operationId: '11111111-1111-4111-8111-111111111111',
      action: 'create',
      namespace: 'mcp-host',
      targetRef: 'mcp-host/host-a',
      operatorSub: 'admin-1',
      requestId: 'request-1',
    })
    const persistHostOutcome = vi.fn()
    setAdministrativeOperationService({ persistHostIntent, persistHostOutcome } as never)
    const customApi = { createNamespacedCustomObject: vi.fn().mockResolvedValue({}) }
    const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })

    try {
      await runWithAdministrativeRequestContext(
        { operatorSub: 'admin-1', requestId: 'request-1' },
        () =>
          service.createResource(
            'hosts',
            {
              metadata: {
                name: 'host-a',
                annotations: {
                  'clerum.io/administrative-intent-id': 'caller-value',
                  'clerum.io/administrative-intent-generation': '999',
                  keep: 'yes',
                },
              },
              spec: {},
            },
            'mcp-host'
          )
      )
    } finally {
      setAdministrativeOperationService(null)
    }

    expect(persistHostIntent).toHaveBeenCalledWith(
      expect.objectContaining({
        operatorSub: 'admin-1',
        requestId: 'request-1',
        name: 'host-a',
      })
    )
    expect(customApi.createNamespacedCustomObject).toHaveBeenCalledWith(
      expect.objectContaining({
        body: expect.objectContaining({
          metadata: expect.objectContaining({
            annotations: {
              keep: 'yes',
              'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
              'clerum.io/administrative-intent-generation': '1',
            },
          }),
        }),
      })
    )
  })

  it('does not create an operator-bound intent outside an authenticated admin context', async () => {
    const persistHostIntent = vi.fn()
    setAdministrativeOperationService({ persistHostIntent } as never)
    const customApi = { createNamespacedCustomObject: vi.fn().mockResolvedValue({}) }
    try {
      await new ResourceService(customApi as never, 'control-plane', {
        hosts: 'mcp-host',
      }).createResource('hosts', { metadata: { name: 'host-a' }, spec: {} }, 'mcp-host')
    } finally {
      setAdministrativeOperationService(null)
    }
    expect(persistHostIntent).not.toHaveBeenCalled()
  })
})

/**
 * #329 — the annotation must name the generation the apiserver PERSISTED, not
 * the one control-api predicted before the write. The Host CRD declares a
 * status subresource, so `metadata.generation` advances only on spec changes:
 * a replace whose spec is unchanged leaves the object at N while the
 * prediction writes N+1, and nothing ever retires that annotation, so the Host
 * stops binding outcomes permanently.
 */
describe('ResourceService administrative intent generation reconciliation (#329)', () => {
  const GENERATION = 'clerum.io/administrative-intent-generation'
  const INTENT_ID = 'clerum.io/administrative-intent-id'
  const OPERATION_ID = '11111111-1111-4111-8111-111111111111'
  const OTHER_OPERATION_ID = '33333333-3333-4333-8333-333333333333'

  function withIntent() {
    setAdministrativeOperationService({
      persistHostIntent: vi.fn().mockResolvedValue({
        operationId: OPERATION_ID,
        action: 'update',
        namespace: 'mcp-host',
        targetRef: 'mcp-host/host-a',
        operatorSub: 'admin-1',
        requestId: 'request-1',
      }),
      persistHostOutcome: vi.fn(),
    } as never)
  }

  /**
   * What the apiserver answers a replace with: the object AS PERSISTED, which
   * therefore echoes back the annotation pair the replace just wrote.
   *
   * Carrying the annotations is not decoration. The correction checks ownership
   * on this object before patching, so a fixture that omitted them would drive
   * a branch the real apiserver can never produce, and every assertion built on
   * it would be measuring the fixture.
   */
  function writtenObject(
    generation: number | undefined,
    resourceVersion: string,
    operationId: string | null = OPERATION_ID
  ) {
    const annotations: Record<string, string> = { [GENERATION]: '5' }
    if (operationId) annotations[INTENT_ID] = operationId
    return {
      metadata: {
        name: 'host-a',
        namespace: 'mcp-host',
        resourceVersion,
        ...(generation === undefined ? {} : { generation }),
        annotations,
      },
      spec: { contextRef: 'next' },
    }
  }

  /** The object as it stands before the write: live generation 4, no intent. */
  function liveObject() {
    return {
      metadata: {
        name: 'host-a',
        namespace: 'mcp-host',
        resourceVersion: '5',
        generation: 4,
        annotations: {},
      },
      spec: { contextRef: 'live' },
    }
  }

  function api(replaced: Record<string, unknown>, patch = vi.fn().mockResolvedValue({})) {
    return {
      getNamespacedCustomObject: vi.fn().mockResolvedValue(liveObject()),
      replaceNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),
      patchNamespacedCustomObject: patch,
      createNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),
      listNamespacedCustomObject: vi.fn(),
    }
  }

  function service(customApi: ReturnType<typeof api>) {
    return new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })
  }

  async function asOperator<T>(run: () => Promise<T>): Promise<T> {
    withIntent()
    try {
      return await runWithAdministrativeRequestContext(
        { operatorSub: 'admin-1', requestId: 'request-1' },
        run
      )
    } finally {
      setAdministrativeOperationService(null)
    }
  }

  async function update(customApi: ReturnType<typeof api>) {
    return asOperator(() =>
      service(customApi).updateResource(
        'hosts',
        'host-a',
        { spec: { contextRef: 'next' } },
        'mcp-host'
      )
    )
  }

  /**
   * Reads the reconcile-failure counter for one reason. Used as a liveness
   * witness that needs no mock: only the branch under test increments its own
   * reason, so a delta of 1 proves that exact branch ran. Counters are
   * process-global and other tests share them, hence the before/after delta
   * rather than an absolute value.
   */
  async function failureCount(reason: string): Promise<number> {
    const { values } = await governedTraceAdministrativeIntentReconcileFailedTotal.get()
    return values
      .filter(value => value.labels.reason === reason)
      .reduce((sum, value) => sum + value.value, 0)
  }

  it('lowers the annotation to the persisted generation when the replace did not bump it', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const customApi = api(writtenObject(4, '6'), patch)

    await update(customApi)

    // Liveness: the replace itself carried the PREDICTED value, so the patch
    // below is a correction of a real mismatch and not a no-op.
    const replaceBody = customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body
    expect(replaceBody.metadata.annotations[GENERATION]).toBe('5')

    expect(patch).toHaveBeenCalledOnce()
    const [patchArgs] = patch.mock.calls[0]!
    expect(patchArgs).toMatchObject({
      namespace: 'mcp-host',
      plural: 'hosts',
      name: 'host-a',
      body: {
        metadata: {
          // The precondition comes from the object the write returned, so a
          // concurrent change between the replace and the patch is refused
          // rather than overwritten.
          resourceVersion: '6',
          annotations: { [GENERATION]: '4' },
        },
      },
    })
    // Annotations only: a patch that carried a spec would bump the generation
    // it is correcting and feed itself.
    expect(Object.keys(patchArgs.body.metadata)).toEqual(['resourceVersion', 'annotations'])
    expect(patchArgs.body).not.toHaveProperty('spec')
  })

  /**
   * A differential, because the equality branch emits nothing observable: no
   * patch, no log, no counter. "The patch did not fire" is therefore satisfied
   * just as well by a reconcile that was never called at all, so the matching
   * case is measured against a mismatching one driven through the SAME service
   * in the same test. The second half is the liveness witness for the first.
   *
   * Deleting the reconcile call entirely takes the second half red, which is
   * exactly the mutation this test previously survived.
   */
  it('patches on a mismatch and not on a match, with the correction armed both times', async () => {
    const matched = vi.fn().mockResolvedValue({})
    await update(api(writtenObject(5, '6'), matched))
    expect(matched).not.toHaveBeenCalled()

    const mismatched = vi.fn().mockResolvedValue({})
    await update(api(writtenObject(4, '6'), mismatched))
    expect(mismatched).toHaveBeenCalledOnce()
    expect(mismatched.mock.calls[0]![0].body.metadata.annotations[GENERATION]).toBe('4')
  })

  /**
   * Ownership, not generation, decides whether the correction is still ours.
   * The two annotation keys are written as a pair, so patching the generation
   * key while another operation owns the id key would fuse that operation's id
   * to our generation — a pair that never existed on any object.
   *
   * Same shape as the test above: the owned case is the witness for the
   * reassigned one, and the reassigned branch emits no counter because it is a
   * correct decision to skip, not a failure to apply.
   */
  it('skips the correction when another operation owns the intent annotation', async () => {
    const ours = vi.fn().mockResolvedValue({})
    await update(api(writtenObject(4, '6', OPERATION_ID), ours))
    expect(ours).toHaveBeenCalledOnce()

    const theirs = vi.fn().mockResolvedValue({})
    await update(api(writtenObject(4, '6', OTHER_OPERATION_ID), theirs))
    expect(theirs).not.toHaveBeenCalled()
  })

  /**
   * An object with no intent id carries no pending obligation, so writing the
   * generation key alone would leave half a pair naming no operation. Distinct
   * from reassignment, and counted under its own reason so the two cannot be
   * confused on a dashboard.
   */
  it('counts an absent intent annotation under its own reason and writes nothing', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const before = await failureCount('intent_absent')

    await update(api(writtenObject(4, '6', null), patch))

    // Witness: the counter this branch alone increments moved by exactly one.
    expect((await failureCount('intent_absent')) - before).toBe(1)
    expect(patch).not.toHaveBeenCalled()
  })

  /**
   * The object moved past the generation this operation produced. Correct it
   * ANYWAY: `persisted` is what this write actually produced, which stays true
   * wherever the object has since got to, and writing it lands
   * `annotated < live` — the terminal refusal the binding resolver exists to
   * make.
   *
   * Leaving the PREDICTION in place is the dangerous option. `predicted` is by
   * construction the next generation the object is most likely to reach, so a
   * concurrent bump can make it match the live generation exactly, and the
   * resolver would then bind another writer's change to this operator's intent
   * — the audit-trail falsification #329 exists to prevent.
   */
  it('corrects to the produced generation even when the object outpaced it', async () => {
    const patch = vi.fn().mockRejectedValueOnce(makeConflictError()).mockResolvedValue({})
    const customApi = api(writtenObject(4, '6'), patch)
    customApi.getNamespacedCustomObject.mockResolvedValueOnce(liveObject()).mockResolvedValue({
      metadata: {
        name: 'host-a',
        namespace: 'mcp-host',
        resourceVersion: '9',
        generation: 6,
        annotations: { [INTENT_ID]: OPERATION_ID, [GENERATION]: '5' },
      },
      spec: { contextRef: 'other' },
    })

    await update(customApi)

    // The retry re-read and patched again — the loop's reason for existing,
    // untested before this. The second patch carries the generation THIS write
    // produced (4), below the live 6, against the re-read's resourceVersion.
    expect(patch).toHaveBeenCalledTimes(2)
    expect(patch.mock.calls[1]![0].body.metadata).toMatchObject({
      resourceVersion: '9',
      annotations: { [GENERATION]: '4' },
    })
  })

  /**
   * Three conflicts exhaust the loop. The write itself succeeded, so the method
   * still returns normally; the reason is reported rather than swallowed.
   */
  it('reports patch exhaustion after the last conflict instead of throwing', async () => {
    const patch = vi.fn().mockRejectedValue(makeConflictError())
    const customApi = api(writtenObject(4, '6'), patch)
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(liveObject())
      .mockResolvedValue(writtenObject(4, '9'))
    const before = await failureCount('patch_failed')

    await expect(update(customApi)).resolves.toMatchObject({ metadata: { name: 'host-a' } })

    expect(patch).toHaveBeenCalledTimes(3)
    expect((await failureCount('patch_failed')) - before).toBe(1)
  })

  /**
   * Without a resourceVersion there is no optimistic-concurrency precondition,
   * and an unconditional patch could overwrite a concurrent write. Skipping is
   * the fail-closed answer; the counter says why.
   */
  it('refuses to patch without a resourceVersion precondition', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const written = writtenObject(4, '6')
    delete (written.metadata as { resourceVersion?: string }).resourceVersion
    const before = await failureCount('no_precondition')

    await update(api(written, patch))

    expect((await failureCount('no_precondition')) - before).toBe(1)
    expect(patch).not.toHaveBeenCalled()
  })

  /**
   * A create is the one prediction that cannot be wrong. `PrepareForCreate`
   * sets `metadata.generation = 1` unconditionally, and it runs AFTER mutating
   * admission, so no webhook can move it. The annotation is therefore written
   * as literal `1` and the correction never fires.
   *
   * The witness is the annotation on the create body: without it, "no patch"
   * would also describe a create that never annotated anything.
   */
  it('annotates a create with the literal generation 1 and corrects nothing', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const customApi = api(
      {
        metadata: {
          name: 'host-a',
          namespace: 'mcp-host',
          generation: 1,
          resourceVersion: '1',
          annotations: { [INTENT_ID]: OPERATION_ID, [GENERATION]: '1' },
        },
      },
      patch
    )

    await asOperator(() =>
      service(customApi).createResource(
        'hosts',
        { metadata: { name: 'host-a' }, spec: { contextRef: 'next' } },
        'mcp-host'
      )
    )

    const createBody = customApi.createNamespacedCustomObject.mock.calls[0]![0].body
    expect(createBody.metadata.annotations[GENERATION]).toBe('1')
    expect(patch).not.toHaveBeenCalled()
  })

  /**
   * `mutateResource` is the likeliest producer of drift in the repository: it
   * issues a replace even when the mutation computes an identical spec, and the
   * Host CRD's status subresource means an unchanged spec does not advance the
   * generation. Covered here because a wrong argument at that call site would
   * otherwise ship green — the update path's tests say nothing about it.
   */
  it('corrects a mutateResource replace that did not advance the generation', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const customApi = api(writtenObject(4, '6'), patch)

    await asOperator(() =>
      service(customApi).mutateResource(
        'hosts',
        'host-a',
        () => ({ spec: { contextRef: 'live' } }),
        'mcp-host'
      )
    )

    expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()
    expect(patch).toHaveBeenCalledOnce()
    expect(patch.mock.calls[0]![0].body.metadata.annotations[GENERATION]).toBe('4')
  })

  /**
   * A re-read that fails leaves the annotation at the prediction. The write
   * succeeded, so the method still returns normally and reports the reason.
   * `extractK8sStatus` reads the status off the wrapped error, which is why it
   * had to learn `httpStatus`: `getResource` rejects with `K8sNotFoundError`,
   * whose status lives on that field alone.
   */
  it('reports a failed re-read instead of throwing, and reads its status', async () => {
    const patch = vi.fn().mockRejectedValue(makeConflictError())
    const customApi = api(writtenObject(4, '6'), patch)
    customApi.getNamespacedCustomObject
      .mockResolvedValueOnce(liveObject())
      .mockRejectedValue(new K8sNotFoundError('hosts/host-a not found in namespace mcp-host'))
    const before = await failureCount('reread_failed')

    await expect(update(customApi)).resolves.toMatchObject({ metadata: { name: 'host-a' } })

    expect(patch).toHaveBeenCalledOnce()
    expect((await failureCount('reread_failed')) - before).toBe(1)
  })

  it('reports rather than throws when the write response carries no generation', async () => {
    const patch = vi.fn().mockResolvedValue({})
    const customApi = api(writtenObject(undefined, '6'), patch)
    const before = await failureCount('generation_missing')

    // The mutation itself succeeded, so it must still return normally: turning
    // an uncorrectable annotation into a caller-visible error would report a
    // completed write as failed.
    await expect(update(customApi)).resolves.toMatchObject({
      metadata: { name: 'host-a' },
    })
    expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()
    // Witness the name promises: the branch REPORTED. Without this the test
    // passes with the whole reconcile call deleted, since "no patch went out"
    // was true before this feature existed.
    expect((await failureCount('generation_missing')) - before).toBe(1)
    expect(patch).not.toHaveBeenCalled()
  })
})