Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
52.1 KB
Raw Blame
/**
 * LlmHook Reconciler — turns `image`-target LlmHook CRs into a shared
 * Deployment + Service + NetworkPolicy in the llm-hooks namespace, and writes
 * LlmHook.status. `service` and `remote` targets deploy nothing (status-only).
 *
 * The one hard divergence from every other HCC reconciler is digest-dedup:
 * this is N-CRs → 1-workload. Multiple `image` hooks that share a POD KEY
 * (image ref + port + envSecret + egressBindings + addCapabilities) co-locate
 * on a single pod and are addressed by distinct spec.path. Three mechanisms
 * replace the single-owner-ref cascade every other reconciler relies on:
 *   1. reference-counted, label-owned workloads (delete when the last member
 *      for a pod key is gone; startup/periodic sweep for orphans);
 *   2. per-pod-key serialization (not per-CR-name — two CRs mutating the same
 *      pod key would otherwise race on the same Deployment/Service/NP);
 *   3. a Host→LlmHook reverse index for the NetworkPolicy ingress set.
 *
 * Mirrors McpServerReconciler (hardened securityContext, credentials-revision
 * annotation, create-then-409 replaceWithConflictRetry, ownership-verifying
 * deletes, status-subresource JSON-Patch writer) and NetworkPolicyReconciler
 * (default-deny + infra baseline, L2 reverse-index ingress, L3 egress + SSRF
 * validators).
 */
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash } from 'crypto'
import { AsyncLocalStorage } from 'node:async_hooks'
import * as dns from 'node:dns/promises'
import { isIP } from 'node:net'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import {
  HOOK_PODKEY_LABEL,
  HOST_LABEL,
  LLMHOOK_LABEL,
  MANAGED_BY_LABEL,
  MANAGED_BY_VALUE,
  POLICY_TYPE_LABEL,
} from './constants'
import { hccLogger } from './logger'
import { createsTotal } from './metrics'
import { isAllowedExternalEgressCidr, isPublicDnsHostname } from './networkPolicyReconciler'
import { HostCRD, LlmHookCRD, LlmHookCondition, LlmHookImageTarget, LlmHookStatus } from './types'
import {
  type ResourceApplyResult,
  accumulateApplyResult,
  deploymentMatchesDesired,
  ensureResource,
  getErrorCode,
  networkPolicyMatchesDesired,
  observeCreate,
  observeExistenceRead,
  preserveDeploymentAnnotations,
  preserveObjectAnnotations,
  preserveServiceAssignedFields,
  replaceWithConflictRetry,
  serviceMatchesDesired,
} from './utils'

/** One apply per LlmHook-owned resource. Not comparable to GFS `objects`. */
type LlmHookResyncCounters = { objects: number; writes: number; skips: number }

const llmHookResyncPass = new AsyncLocalStorage<LlmHookResyncCounters>()

function recordLlmHookApply(result: ResourceApplyResult): void {
  const acc = llmHookResyncPass.getStore()
  if (!acc) return
  accumulateApplyResult(acc, result)
}

const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_LLMHOOKS = 'llmhooks'
const CREDENTIALS_REVISION_ANNOTATION = 'clerum.io/credentials-revision'
const LOG = '[LlmHook]'

/** Machine-readable Ready condition reasons (bounded enum, §6). */
export type LlmHookReadyReason =
  | 'NoWorkload'
  | 'Deploying'
  | 'Ready'
  | 'DuplicatePath'
  | 'SecretNotFound'
  | 'InvalidEgress'
  | 'ReconcileError'

type LlmHookReconcilerDeps = {
  appsApi?: k8s.AppsV1Api
  coreApi?: k8s.CoreV1Api
  customApi?: k8s.CustomObjectsApi
  networkingApi?: k8s.NetworkingV1Api
}

type SecretValidation = { ok: true; revision: string } | { ok: false; message: string }

type EgressBuild = { rules: k8s.V1NetworkPolicyEgressRule[]; failures: string[] }

type StatusExtras = {
  observedDigest?: string
  readyReplicas?: number
  lastReconciled?: string
}

/**
 * Collect every installed-hook id a Host references across all guardrail
 * lifecycle phases. This is the Host side of the Host→LlmHook reverse index
 * that drives the shared pod's NetworkPolicy ingress set (§5).
 */
export function referencedHookIds(host: HostCRD | undefined): string[] {
  const hooks = host?.spec.guardrails?.hooks
  if (!hooks) return []
  const ids: string[] = []
  for (const phase of Object.values(hooks)) {
    if (!Array.isArray(phase)) continue
    for (const ref of phase) {
      if (ref && typeof ref.id === 'string' && ref.id) ids.push(ref.id)
    }
  }
  return ids
}

/**
 * Stable pod key over the POD-LEVEL fields of an image target (§1.1). Two
 * hooks with the same key safely share a pod; a different envSecret or
 * egressBindings gets its own key → its own pod (a different trust/egress
 * boundary). path/order/capabilities/failMode/config are NOT in the key.
 * Returns null for non-image (service/remote) targets.
 */
export function computePodKey(hook: LlmHookCRD): string | null {
  const img = hook.spec.target?.image
  if (!img) return null
  const canonical = {
    imageRef: img.ref,
    port: img.port,
    envSecret: img.envSecret ?? '',
    egressBindings: normalizeEgressBindings(img.egressBindings),
    addCapabilities: [...(img.security?.addCapabilities ?? [])].sort(),
  }
  return createHash('sha256').update(JSON.stringify(canonical)).digest('hex').slice(0, 16)
}

/** Canonicalize egressBindings into a stable, order-independent form. */
function normalizeEgressBindings(
  bindings: LlmHookImageTarget['egressBindings']
): Array<{ cidr: string | null; toFQDN: string | null; ports: number[] }> {
  return (bindings ?? [])
    .map(b => ({
      cidr: b.cidr ?? null,
      toFQDN: b.toFQDN ?? null,
      ports: [...(b.ports ?? [])].sort((a, z) => a - z),
    }))
    .sort((a, z) => JSON.stringify(a).localeCompare(JSON.stringify(z)))
}

export function podKeyResourceName(podKey: string): string {
  return `llmhook-${podKey}`
}

/** NetworkPolicy name for a service-target hook's per-CR ingress policy (§8.2). */
export function serviceTargetNpName(crName: string): string {
  return `llmhook-svc-${crName}`
}

type TargetKind = 'image' | 'service' | 'remote' | 'invalid'

function classifyTarget(hook: LlmHookCRD): TargetKind {
  const t = hook.spec.target ?? {}
  if (t.image) return 'image'
  if (t.service) return 'service'
  if (t.remote) return 'remote'
  return 'invalid'
}

export class LlmHookReconciler {
  private readonly appsApi: k8s.AppsV1Api
  private readonly coreApi: k8s.CoreV1Api
  private readonly customApi: k8s.CustomObjectsApi
  private readonly networkingApi: k8s.NetworkingV1Api

  /** Cache of LlmHook CRs by name (owned by the watcher). */
  private readonly hooks: Map<string, LlmHookCRD>
  /** Cache of Host CRs by name (owned by the watcher) — reverse-index source. */
  private readonly hosts: Map<string, HostCRD>

  /**
   * Tail promise of the in-flight work per SERIALIZATION KEY. Image targets
   * serialize by pod key so all work touching one shared workload runs strictly
   * sequentially while different pod keys run concurrently (§4); service/remote
   * targets serialize by `name:<crName>`.
   */
  private readonly inFlight: Map<string, Promise<void>> = new Map()

  constructor(
    kc: k8s.KubeConfig,
    hookCache: Map<string, LlmHookCRD>,
    hostCache: Map<string, HostCRD>,
    deps?: LlmHookReconcilerDeps
  ) {
    this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
    this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
    this.customApi = deps?.customApi ?? kc.makeApiClient(k8s.CustomObjectsApi)
    this.networkingApi = deps?.networkingApi ?? kc.makeApiClient(k8s.NetworkingV1Api)
    this.hooks = hookCache
    this.hosts = hostCache
  }

  // ─── Serialization ──────────────────────────────────────────────────

  private runSerialized(key: string, fn: () => Promise<void>): Promise<void> {
    const prev = this.inFlight.get(key) ?? Promise.resolve()
    const next = prev.catch(() => undefined).then(fn)
    this.inFlight.set(key, next)
    return (async () => {
      try {
        await next
      } finally {
        if (this.inFlight.get(key) === next) this.inFlight.delete(key)
      }
    })()
  }

  // ─── Public API ─────────────────────────────────────────────────────

  /**
   * Reconcile a created or modified LlmHook. `previousPodKey` (the pod key the
   * CR had before this event, computed from the cached object) lets an image
   * bump chain teardown of the old workload on the old key AND ensure the new
   * one on the new key (§4), so neither interleaves with the other.
   */
  async reconcile(hook: LlmHookCRD, previousPodKey?: string | null): Promise<void> {
    const kind = classifyTarget(hook)
    if (kind !== 'image') {
      await this.runSerialized(`name:${hook.name}`, () => this.reconcileNonImage(hook, kind))
      return
    }
    // Clean any stale service-target ingress NP left by a prior service→image flip.
    await this.deleteServiceTargetNetworkPolicy(hook.name)
    const podKey = computePodKey(hook)!
    if (previousPodKey && previousPodKey !== podKey) {
      // The image target changed pod keys: tear down the stale one (if it now
      // has zero members) before/while ensuring the new one.
      await this.runSerialized(previousPodKey, () => this.reconcilePodKey(previousPodKey))
    }
    await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))
  }

  /**
   * Reconcile a deleted LlmHook. The cached object was already evicted by the
   * watcher; `formerPodKey` is its pod key (computed before eviction). Running
   * the pod-key reconcile now sees a smaller (possibly empty) member set and
   * GCs the workload when the last member is gone (§3).
   */
  async reconcileDelete(name: string, formerPodKey?: string | null): Promise<void> {
    if (!formerPodKey) {
      // service/remote (or unknown) — no workload was deployed, but a service
      // target may have left an ingress NP; remove it.
      await this.runSerialized(`name:${name}`, () => this.deleteServiceTargetNetworkPolicy(name))
      return
    }
    await this.runSerialized(formerPodKey, () => this.reconcilePodKey(formerPodKey))
  }

  /**
   * Re-reconcile only the NetworkPolicy ingress for the pod keys referenced by
   * the given hook ids. Called from the Host-watch fan-out so adding/removing a
   * Host reference re-computes which mcp-hosts the shared pod admits (§5).
   */
  async reconcileNetworkPoliciesForHooks(hookNames: string[]): Promise<void> {
    const podKeys = new Set<string>()
    for (const name of hookNames) {
      const hook = this.hooks.get(name)
      if (!hook) continue
      // Service targets: refresh the per-CR ingress NP so the admitted-host set
      // tracks the Host reference change (image targets handled by pod key below).
      if (classifyTarget(hook) === 'service') {
        await this.runSerialized(`name:${name}`, () => this.ensureServiceTargetNetworkPolicy(hook))
        continue
      }
      const key = computePodKey(hook)
      if (key) podKeys.add(key)
    }
    for (const podKey of podKeys) {
      await this.runSerialized(podKey, async () => {
        const members = this.membersForPodKey(podKey)
        if (members.length === 0) return
        const img = members[0].spec.target.image!
        const egress = await this.buildEgressRules(img)
        // A now-invalid egress binding must not silently re-open the pod; the
        // ingress refresh still applies (fail-closed egress on the next full
        // reconcile). Apply ingress with whatever egress currently validates.
        await this.ensureNetworkPolicy(podKey, members, egress.failures.length ? [] : egress.rules)
      })
    }

    // Refresh the per-host scoped EGRESS for every host referencing an affected
    // hook, so a Host reference change (or an image bump that moves a pod key)
    // keeps the host's egress allowlist in sync with what it may reach (N1/N7).
    // The "host dropped its last reference" case is covered by the Host-watch
    // calling reconcileHostEgress(host) directly.
    const affectedHosts = new Map<string, HostCRD>()
    for (const name of hookNames) {
      const hook = this.hooks.get(name)
      if (!hook) continue
      for (const h of this.referencingHosts([hook])) affectedHosts.set(h.name, h)
    }
    for (const host of affectedHosts.values()) {
      await this.reconcileHostEgress(host)
    }
  }

  /**
   * Full reconciliation pass (startup + periodic resync). Reconciles every
   * distinct pod key once, every non-image hook, then sweeps label-orphaned
   * `llmhook-*` workloads whose pod key has zero live members (§3).
   */
  async fullReconcile(hooks: LlmHookCRD[]): Promise<void> {
    const started = Date.now()
    const acc: LlmHookResyncCounters = { objects: 0, writes: 0, skips: 0 }
    hccLogger.info('Running full reconciliation', { scope: 'LlmHook', hooks: hooks.length })
    await llmHookResyncPass.run(acc, async () => {
      const imagePodKeys = new Set<string>()
      for (const hook of hooks) {
        const kind = classifyTarget(hook)
        if (kind === 'image') {
          const key = computePodKey(hook)
          if (key) imagePodKeys.add(key)
        } else {
          await this.reconcile(hook)
        }
      }
      for (const podKey of imagePodKeys) {
        await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))
      }
      // Converge every Host's scoped egress-to-hooks policy (N1/N7) — the periodic
      // backstop for a missed watch event or an image bump that moved a pod key.
      for (const host of this.hosts.values()) {
        await this.reconcileHostEgress(host)
      }
      await this.sweepOrphanedWorkloads()
    })
    hccLogger.info('Resync pass completed', {
      scope: 'LlmHook',
      objects: acc.objects,
      writes: acc.writes,
      skips: acc.skips,
      passMs: Date.now() - started,
    })
  }

  // ─── Members / reverse index ────────────────────────────────────────

  /** All cached image-target LlmHooks whose pod key equals `podKey`. */
  private membersForPodKey(podKey: string): LlmHookCRD[] {
    return [...this.hooks.values()]
      .filter(h => classifyTarget(h) === 'image' && computePodKey(h) === podKey)
      .sort((a, b) => a.name.localeCompare(b.name))
  }

  /** Hosts whose guardrails reference any of the given member hooks (§5). */
  private referencingHosts(members: LlmHookCRD[]): HostCRD[] {
    const memberNames = new Set(members.map(m => m.name))
    return [...this.hosts.values()]
      .filter(h => referencedHookIds(h).some(id => memberNames.has(id)))
      .sort((a, b) => a.name.localeCompare(b.name))
  }

  // ─── Core pod-key reconcile ─────────────────────────────────────────

  private async reconcilePodKey(podKey: string): Promise<void> {
    const members = this.membersForPodKey(podKey)

    // Reference count == live members. Zero → tear the workload down (§3).
    if (members.length === 0) {
      await this.gcPodKey(podKey)
      return
    }

    const rep = members[0]
    const img = rep.spec.target.image!

    // Path uniqueness within the pod key (§1.2): losers fail closed, winners
    // are still served.
    const duplicatePathNames = this.duplicatePathMembers(members)

    // Secret gate (mirror McpServer secret gate).
    const secret = await this.validateSecret(img)
    if (!secret.ok) {
      for (const m of members) {
        await this.writeStatus(m, condition('False', 'SecretNotFound', secret.message))
      }
      return
    }

    // Egress hardening (§5 caveat): apply McpServer-grade SSRF/private-range
    // validation to the looser LlmHook.egressBindings; a rejected binding fails
    // the whole pod key closed (egress is a pod-level boundary).
    const egress = await this.buildEgressRules(img)
    if (egress.failures.length > 0) {
      const message = egress.failures.join('; ')
      for (const m of members) {
        await this.writeStatus(m, condition('False', 'InvalidEgress', message))
      }
      return
    }

    // Ensure the shared workload (idempotent create-then-409 replace).
    try {
      await this.ensureDeployment(podKey, members, secret.revision)
      await this.ensureService(podKey, img.port)
      await this.ensureNetworkPolicy(podKey, members, egress.rules)
    } catch (err) {
      const message = `Resource sync failed: ${errMsg(err)}`
      for (const m of members) {
        await this.writeStatus(m, condition('False', 'ReconcileError', message))
      }
      return
    }

    // Readiness + observedDigest → per-member status (each member is a distinct
    // CR and gets its own status even though they share a pod, §6).
    const name = podKeyResourceName(podKey)
    const rollout = await this.readDeploymentRollout(name)
    const observedDigest = await this.readObservedDigest(podKey)
    const nowIso = new Date().toISOString()
    const extras: StatusExtras = {
      observedDigest,
      readyReplicas: rollout.readyReplicas,
      ...(rollout.ready ? { lastReconciled: nowIso } : {}),
    }

    for (const m of members) {
      if (duplicatePathNames.has(m.name)) {
        await this.writeStatus(
          m,
          condition(
            'False',
            'DuplicatePath',
            `spec.path "${m.spec.path ?? '/'}" collides with another co-located hook sharing this pod`
          ),
          extras
        )
        continue
      }
      await this.writeStatus(
        m,
        rollout.ready
          ? condition('True', 'Ready', 'Hook deployment ready')
          : condition('Unknown', 'Deploying', `Waiting for hook pod: ${rollout.detail}`),
        extras
      )
    }
  }

  private async reconcileNonImage(hook: LlmHookCRD, kind: TargetKind): Promise<void> {
    // service / remote targets deploy nothing: mcp-host dials them directly.
    if (kind === 'invalid') {
      await this.writeStatus(
        hook,
        condition('False', 'ReconcileError', 'target must be exactly one of image, service, remote')
      )
      return
    }
    // A service target gets an ingress NP admitting only referencing hosts; a
    // remote target has no in-cluster pod, so clear any stale NP (e.g. after a
    // service→remote target flip).
    if (kind === 'service') {
      await this.ensureServiceTargetNetworkPolicy(hook)
    } else {
      await this.deleteServiceTargetNetworkPolicy(hook.name)
    }
    await this.writeStatus(
      hook,
      condition('True', 'NoWorkload', 'No workload deployed for service/remote target'),
      { lastReconciled: new Date().toISOString() }
    )
  }

  /**
   * Names of members that lose a path collision. The first member (by sorted
   * name) to claim a path wins; later members claiming the same path are
   * flagged DuplicatePath and not routed.
   */
  private duplicatePathMembers(members: LlmHookCRD[]): Set<string> {
    const seen = new Map<string, string>()
    const losers = new Set<string>()
    for (const m of members) {
      const path = m.spec.path ?? '/'
      if (seen.has(path)) {
        losers.add(m.name)
      } else {
        seen.set(path, m.name)
      }
    }
    return losers
  }

  // ─── Secret validation (mirror validateSecret, credentials revision) ──

  private async validateSecret(img: LlmHookImageTarget): Promise<SecretValidation> {
    if (!img.envSecret) return { ok: true, revision: '' }
    try {
      const secret = await this.coreApi.readNamespacedSecret({
        name: img.envSecret,
        namespace: config.llmHooksNamespace,
      })
      const data = secret.data || {}
      const revision = createHash('sha256')
        .update(
          JSON.stringify(
            Object.keys(data)
              .sort()
              .map(k => [k, data[k]])
          )
        )
        .digest('hex')
      return { ok: true, revision }
    } catch (error) {
      const code = getErrorCode(error)
      if (code === 404) {
        return {
          ok: false,
          message: `Secret "${img.envSecret}" not found in namespace "${config.llmHooksNamespace}"`,
        }
      }
      return {
        ok: false,
        message: `Failed to read Secret "${img.envSecret}": ${errMsg(error)}`,
      }
    }
  }

  // ─── Egress builder (reuse NP reconciler's SSRF/private-range validators) ──

  private async buildEgressRules(img: LlmHookImageTarget): Promise<EgressBuild> {
    const rules: k8s.V1NetworkPolicyEgressRule[] = []
    const failures: string[] = []
    for (const binding of img.egressBindings ?? []) {
      const ports: k8s.V1NetworkPolicyPort[] =
        binding.ports && binding.ports.length > 0
          ? binding.ports.map(p => ({ port: p, protocol: 'TCP' as const }))
          : [
              { port: 443, protocol: 'TCP' },
              { port: 80, protocol: 'TCP' },
            ]

      if (binding.cidr) {
        if (!isAllowedExternalEgressCidr(binding.cidr)) {
          failures.push(
            `CIDR "${binding.cidr}" overlaps private, metadata, link-local, multicast, documentation, or reserved ranges`
          )
          continue
        }
        rules.push({ to: [{ ipBlock: { cidr: binding.cidr } }], ports })
        continue
      }

      if (binding.toFQDN) {
        if (!isPublicDnsHostname(binding.toFQDN)) {
          failures.push(
            `hostname "${binding.toFQDN}" is private, internal, metadata, local, or otherwise disallowed`
          )
          continue
        }
        try {
          const ips = [...new Set(await dns.resolve4(binding.toFQDN))].sort()
          if (ips.length === 0) {
            failures.push(`hostname "${binding.toFQDN}" resolved to no IPv4 addresses`)
            continue
          }
          const cidrs = ips.map(ip => `${ip}/32`)
          const disallowed = cidrs.filter(
            c => isIP(c.split('/')[0]) !== 4 || !isAllowedExternalEgressCidr(c)
          )
          if (disallowed.length > 0) {
            failures.push(
              `hostname "${binding.toFQDN}" resolved disallowed address(es): ${disallowed.join(', ')}`
            )
            continue
          }
          for (const cidr of cidrs) rules.push({ to: [{ ipBlock: { cidr } }], ports })
        } catch (err) {
          failures.push(`failed to resolve hostname "${binding.toFQDN}": ${errMsg(err)}`)
        }
        continue
      }

      failures.push('egress binding must declare toFQDN or cidr')
    }

    // Scoped DNS (N5): a hook that declares outbound egress must resolve its
    // targets at runtime, so grant egress to CoreDNS (kube-system:53) — but ONLY
    // for hooks that opted into egress. A pure /v1 responder (no egressBindings)
    // gets no egress at all, "no implicit DNS".
    if (rules.length > 0) {
      rules.push({
        to: [
          { namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' } } },
        ],
        ports: [
          { port: 53, protocol: 'UDP' },
          { port: 53, protocol: 'TCP' },
        ],
      })
    }
    return { rules, failures }
  }

  // ─── Resource builders ──────────────────────────────────────────────

  private podKeyLabels(podKey: string): Record<string, string> {
    return {
      app: podKeyResourceName(podKey),
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [HOOK_PODKEY_LABEL]: podKey,
    }
  }

  private buildDeployment(
    podKey: string,
    members: LlmHookCRD[],
    credentialsRevision: string
  ): k8s.V1Deployment {
    const img = members[0].spec.target.image!
    const name = podKeyResourceName(podKey)
    const labels = this.podKeyLabels(podKey)

    // Defense in depth: the CRD enum already bounds addCapabilities to the
    // workflow-recipe allowlist, but strip anything outside it at reconcile too.
    const addCapabilities = (img.security?.addCapabilities ?? []).filter(cap => {
      if (isWorkflowRecipeDefaultAllowedCapability(cap)) return true
      hccLogger.warn('Forbidden capability stripped', { scope: LOG, podKey, capability: cap })
      return false
    })

    const hardenedContainerSecurityContext: k8s.V1SecurityContext = {
      allowPrivilegeEscalation: false,
      runAsNonRoot: true,
      runAsUser: 1000,
      runAsGroup: 1000,
      capabilities: {
        drop: ['ALL'],
        ...(addCapabilities.length ? { add: addCapabilities } : {}),
      },
      seccompProfile: { type: 'RuntimeDefault' },
      // N8: the image filesystem is immutable to the running hook. Scratch space
      // is the ephemeral `/tmp` emptyDir mounted below — never the image layers.
      readOnlyRootFilesystem: true,
    }

    return {
      apiVersion: 'apps/v1',
      kind: 'Deployment',
      metadata: { name, namespace: config.llmHooksNamespace, labels },
      spec: {
        replicas: 1,
        selector: { matchLabels: { app: name } },
        template: {
          metadata: {
            labels,
            // Rotating the hook's envSecret changes this digest → pod template
            // → rolling restart (mirror McpServer credentials-revision).
            ...(credentialsRevision && {
              annotations: { [CREDENTIALS_REVISION_ANNOTATION]: credentialsRevision },
            }),
          },
          spec: {
            automountServiceAccountToken: false,
            enableServiceLinks: false,
            securityContext: {
              runAsNonRoot: true,
              runAsUser: 1000,
              runAsGroup: 1000,
              seccompProfile: { type: 'RuntimeDefault' },
            },
            ...(img.imagePullSecrets?.length
              ? { imagePullSecrets: img.imagePullSecrets.map(n => ({ name: n })) }
              : {}),
            containers: [
              {
                name: 'hook',
                image: img.ref,
                imagePullPolicy: config.mcpServerImagePullPolicy,
                ports: [{ name: 'http', containerPort: img.port, protocol: 'TCP' }],
                // envSecret injects the hook's OWN credentials (whole Secret).
                ...(img.envSecret ? { envFrom: [{ secretRef: { name: img.envSecret } }] } : {}),
                securityContext: hardenedContainerSecurityContext,
                // Writable scratch under a read-only root fs (N8).
                volumeMounts: [{ name: 'tmp', mountPath: '/tmp' }],
                livenessProbe: {
                  tcpSocket: { port: 'http' as unknown as IntOrString },
                  initialDelaySeconds: 10,
                  periodSeconds: 15,
                },
                readinessProbe: {
                  tcpSocket: { port: 'http' as unknown as IntOrString },
                  initialDelaySeconds: 5,
                  periodSeconds: 10,
                },
                resources: {
                  requests: { memory: '64Mi', cpu: '50m' },
                  limits: { memory: '256Mi', cpu: '500m' },
                },
              },
            ],
            volumes: [{ name: 'tmp', emptyDir: {} }],
          },
        },
      },
    }
  }

  private buildService(podKey: string, port: number): k8s.V1Service {
    const name = podKeyResourceName(podKey)
    return {
      apiVersion: 'v1',
      kind: 'Service',
      metadata: { name, namespace: config.llmHooksNamespace, labels: this.podKeyLabels(podKey) },
      spec: {
        type: 'ClusterIP',
        ports: [{ port, targetPort: 'http' as IntOrString, protocol: 'TCP', name: 'http' }],
        selector: { app: name },
      },
    }
  }

  private buildNetworkPolicy(
    podKey: string,
    members: LlmHookCRD[],
    egressRules: k8s.V1NetworkPolicyEgressRule[]
  ): k8s.V1NetworkPolicy {
    const name = podKeyResourceName(podKey)
    const port = members[0].spec.target.image!.port
    const hosts = this.referencingHosts(members)

    // Ingress admits EXACTLY the mcp-hosts whose Host CR references any
    // co-located member (Host→LlmHook reverse index, §5). No referencing host
    // ⇒ empty ingress ⇒ deny all (fail-closed until a Host references it).
    const ingress: k8s.V1NetworkPolicyIngressRule[] =
      hosts.length > 0
        ? [
            {
              _from: hosts.map(h => ({
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.hostNamespace },
                },
                podSelector: { matchLabels: { [HOST_LABEL]: h.name } },
              })),
              ports: [{ port, protocol: 'TCP' }],
            },
          ]
        : []

    const policyTypes: Array<'Ingress' | 'Egress'> = ['Ingress']
    if (egressRules.length > 0) policyTypes.push('Egress')

    return {
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name,
        namespace: config.llmHooksNamespace,
        labels: this.podKeyLabels(podKey),
        // Member CR names are recorded as an ANNOTATION, not a label: co-located
        // members are comma-joined, and commas are invalid in a label value (the
        // apiserver 422s), which would break exactly the multi-member digest-dedup
        // case. Annotations allow commas and long values.
        annotations: { 'clerum.io/llmhook-members': members.map(m => m.name).join(',') },
      },
      spec: {
        podSelector: { matchLabels: { [HOOK_PODKEY_LABEL]: podKey } },
        policyTypes,
        ingress,
        ...(egressRules.length > 0 ? { egress: egressRules } : {}),
      },
    }
  }

  // ─── Read-first resource convergence ───────────────────────────────

  private async ensureDeployment(
    podKey: string,
    members: LlmHookCRD[],
    credentialsRevision: string
  ): Promise<void> {
    const deployment = this.buildDeployment(podKey, members, credentialsRevision)
    const name = deployment.metadata!.name!
    recordLlmHookApply(
      await ensureResource({
        read: () =>
          observeExistenceRead('Deployment', () =>
            this.appsApi.readNamespacedDeployment({
              name,
              namespace: config.llmHooksNamespace,
            })
          ),
        create: async () => {
          await observeCreate('Deployment', () =>
            this.appsApi.createNamespacedDeployment({
              namespace: config.llmHooksNamespace,
              body: deployment,
            })
          )
          hccLogger.info('Deployment created', {
            scope: LOG,
            deployment: name,
            namespace: config.llmHooksNamespace,
          })
        },
        converge: read =>
          replaceWithConflictRetry({
            description: `Deployment "${name}"`,
            logPrefix: LOG,
            body: deployment,
            mergeExisting: preserveDeploymentAnnotations,
            isUpToDate: deploymentMatchesDesired,
            read,
            replace: body =>
              this.appsApi.replaceNamespacedDeployment({
                name,
                namespace: config.llmHooksNamespace,
                body,
              }),
          }),
        onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
      })
    )
  }

  private async ensureService(podKey: string, port: number): Promise<void> {
    const service = this.buildService(podKey, port)
    const name = service.metadata!.name!
    recordLlmHookApply(
      await ensureResource({
        read: () =>
          observeExistenceRead('Service', () =>
            this.coreApi.readNamespacedService({
              name,
              namespace: config.llmHooksNamespace,
            })
          ),
        create: async () => {
          await observeCreate('Service', () =>
            this.coreApi.createNamespacedService({
              namespace: config.llmHooksNamespace,
              body: service,
            })
          )
          hccLogger.info('Service created', {
            scope: LOG,
            service: name,
            namespace: config.llmHooksNamespace,
          })
        },
        converge: read =>
          replaceWithConflictRetry({
            description: `Service "${name}"`,
            logPrefix: LOG,
            body: service,
            mergeExisting: preserveServiceAssignedFields,
            isUpToDate: serviceMatchesDesired,
            read,
            replace: body =>
              this.coreApi.replaceNamespacedService({
                name,
                namespace: config.llmHooksNamespace,
                body,
              }),
          }),
        onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
      })
    )
  }

  private async ensureNetworkPolicy(
    podKey: string,
    members: LlmHookCRD[],
    egressRules: k8s.V1NetworkPolicyEgressRule[]
  ): Promise<void> {
    await this.applyNetworkPolicy(this.buildNetworkPolicy(podKey, members, egressRules))
  }

  /** Read first, preserving the existing NetworkPolicy convergence policy. */
  private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {
    const name = policy.metadata!.name!
    const namespace = policy.metadata!.namespace ?? config.llmHooksNamespace
    recordLlmHookApply(
      await ensureResource({
        read: () =>
          observeExistenceRead('NetworkPolicy', () =>
            this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
          ),
        create: async () => {
          await observeCreate('NetworkPolicy', () =>
            this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
          )
          hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })
        },
        converge: read =>
          replaceWithConflictRetry({
            description: `NetworkPolicy "${name}"`,
            logPrefix: LOG,
            body: policy,
            mergeExisting: preserveObjectAnnotations,
            isUpToDate: networkPolicyMatchesDesired,
            read,
            replace: body =>
              this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
          }),
        onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
      })
    )
  }

  /**
   * Per-service-target ingress NP (§8.2). A `service` target deploys no workload,
   * so it never gets a per-podKey policy — with the namespace default-deny that
   * leaves its pods fail-closed. This admits EXACTLY the referencing mcp-hosts to
   * the target Service's pods (empty `ingress` ⇒ deny-all), the service-target
   * analog of the image reverse-index (`buildNetworkPolicy`). Scoped to Services
   * in the llm-hooks namespace, where HCC has RBAC + the default-deny baseline; a
   * Service elsewhere is left to that namespace's own policy (warned), and a
   * missing/selector-less Service is left fail-closed under the default-deny.
   */
  private async ensureServiceTargetNetworkPolicy(hook: LlmHookCRD): Promise<void> {
    const svc = hook.spec.target?.service
    if (!svc?.name || !svc.namespace || !svc.port) return
    if (svc.namespace !== config.llmHooksNamespace) {
      hccLogger.warn('Service-target hook points outside llm-hooks; ingress not enforced', {
        scope: LOG,
        hook: hook.name,
        serviceNamespace: svc.namespace,
        service: svc.name,
      })
      return
    }

    // A missing Service or an empty selector would leave us unable to scope the
    // ingress (an empty podSelector selects ALL pods) — refuse and leave the pods
    // fail-closed under the namespace default-deny instead.
    const selector = await this.readServiceSelector(svc.name, svc.namespace)
    if (!selector) {
      hccLogger.warn('Service-target hook missing selector; leaving fail-closed', {
        scope: LOG,
        hook: hook.name,
        serviceNamespace: svc.namespace,
        service: svc.name,
      })
      return
    }

    const hosts = this.referencingHosts([hook])
    const labels: Record<string, string> = {
      [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
      [POLICY_TYPE_LABEL]: 'service-hook-ingress',
    }
    // Per-CR traceability label — omit if the CR name is not a valid label value.
    if (hook.name.length <= 63) labels[LLMHOOK_LABEL] = hook.name

    const ingress: k8s.V1NetworkPolicyIngressRule[] =
      hosts.length > 0
        ? [
            {
              _from: hosts.map(h => ({
                namespaceSelector: {
                  matchLabels: { 'kubernetes.io/metadata.name': config.hostNamespace },
                },
                podSelector: { matchLabels: { [HOST_LABEL]: h.name } },
              })),
              ports: [{ port: svc.port, protocol: 'TCP' as const }],
            },
          ]
        : []

    await this.applyNetworkPolicy({
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: serviceTargetNpName(hook.name),
        namespace: config.llmHooksNamespace,
        labels,
      },
      spec: {
        podSelector: { matchLabels: selector },
        policyTypes: ['Ingress'],
        ingress,
      },
    })
  }

  /** Delete a service-target hook's ingress NP (CR delete / target-kind change). */
  private async deleteServiceTargetNetworkPolicy(crName: string): Promise<void> {
    const name = serviceTargetNpName(crName)
    const ns = config.llmHooksNamespace
    try {
      const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
      if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {
        await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
        hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name })
      } else {
        hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {
          scope: LOG,
          policy: name,
        })
      }
    } catch (error) {
      if (getErrorCode(error) !== 404) throw error
    }
  }

  /** Read a Service's selector; undefined if missing/empty (can't scope pods). */
  private async readServiceSelector(
    name: string,
    namespace: string
  ): Promise<Record<string, string> | undefined> {
    try {
      const svc = await this.coreApi.readNamespacedService({ name, namespace })
      const selector = svc.spec?.selector
      return selector && Object.keys(selector).length > 0 ? selector : undefined
    } catch (error) {
      if (getErrorCode(error) === 404) return undefined
      throw error
    }
  }

  // ─── Per-host egress reverse-index (§8.2) ───────────────────────────

  private hostEgressNpName(hostName: string): string {
    return `mcp-host-${hostName}-egress-llm-hooks`
  }

  /**
   * Scoped egress reverse-index (§8.2) — the SOURCE-side mirror of the hook
   * ingress policies. A host pod may egress ONLY to the specific hook pods its
   * Host CR references (one rule per referenced in-cluster hook), so a host now
   * reaches only its CRD's hooks from BOTH ends rather than the whole llm-hooks
   * namespace. Serialized per host so concurrent Host/LlmHook events don't race.
   */
  async reconcileHostEgress(host: HostCRD): Promise<void> {
    await this.runSerialized(`host-egress:${host.name}`, () =>
      this.ensureHostEgressNetworkPolicy(host)
    )
  }

  private async ensureHostEgressNetworkPolicy(host: HostCRD): Promise<void> {
    const rules = await this.buildHostEgressRules(host)
    if (rules.length === 0) {
      // No referenced in-cluster hooks → the host has no reason to reach
      // llm-hooks; remove the policy (DNS/control-plane egress live elsewhere).
      await this.deleteHostEgressNetworkPolicy(host.name)
      return
    }
    await this.applyNetworkPolicy({
      apiVersion: 'networking.k8s.io/v1',
      kind: 'NetworkPolicy',
      metadata: {
        name: this.hostEgressNpName(host.name),
        namespace: config.hostNamespace,
        labels: {
          [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
          [HOST_LABEL]: host.name,
          [POLICY_TYPE_LABEL]: 'llm-hooks-egress',
        },
      },
      spec: {
        podSelector: {
          matchLabels: { [HOST_LABEL]: host.name, [MANAGED_BY_LABEL]: MANAGED_BY_VALUE },
        },
        policyTypes: ['Egress'],
        egress: rules,
      },
    })
  }

  /**
   * One egress rule per referenced in-cluster hook (deduped by pod selector +
   * port): image → `hook-pod-key` + `img.port`; service (in llm-hooks) → the
   * target Service's selector + port. Dangling refs, cross-namespace service
   * targets, and `remote` targets add no rule.
   */
  private async buildHostEgressRules(host: HostCRD): Promise<k8s.V1NetworkPolicyEgressRule[]> {
    const llmHooksNs = config.llmHooksNamespace
    const rules: k8s.V1NetworkPolicyEgressRule[] = []
    const seen = new Set<string>()
    for (const id of new Set(referencedHookIds(host))) {
      const hook = this.hooks.get(id)
      if (!hook) continue // dangling reference

      let podSelector: Record<string, string> | undefined
      let port: number | undefined
      const kind = classifyTarget(hook)
      if (kind === 'image') {
        podSelector = { [HOOK_PODKEY_LABEL]: computePodKey(hook)! }
        port = hook.spec.target!.image!.port
      } else if (kind === 'service') {
        const svc = hook.spec.target!.service!
        if (svc.namespace !== llmHooksNs) continue // only llm-hooks is scoped (matches ingress)
        const selector = await this.readServiceSelector(svc.name, svc.namespace)
        if (!selector) continue
        podSelector = selector
        port = svc.port
      } else {
        continue // remote / invalid — not an in-cluster llm-hooks destination
      }

      const dedup = `${JSON.stringify(podSelector)}:${port}`
      if (seen.has(dedup)) continue
      seen.add(dedup)
      rules.push({
        to: [
          {
            namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': llmHooksNs } },
            podSelector: { matchLabels: podSelector },
          },
        ],
        ports: [{ port: port!, protocol: 'TCP' }],
      })
    }
    return rules
  }

  private async deleteHostEgressNetworkPolicy(hostName: string): Promise<void> {
    const name = this.hostEgressNpName(hostName)
    const ns = config.hostNamespace
    try {
      const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
      if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {
        await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
        hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name, namespace: ns })
      }
    } catch (error) {
      if (getErrorCode(error) !== 404) throw error
    }
  }

  // ─── Reference-counted, label-owned GC (§3) ─────────────────────────

  private isHccOwned(
    resource: { metadata?: { labels?: Record<string, string> } },
    podKey: string
  ): boolean {
    const labels = resource.metadata?.labels ?? {}
    return labels[MANAGED_BY_LABEL] === MANAGED_BY_VALUE && labels[HOOK_PODKEY_LABEL] === podKey
  }

  private async gcPodKey(podKey: string): Promise<void> {
    const name = podKeyResourceName(podKey)
    const ns = config.llmHooksNamespace
    // Deployment
    try {
      const dep = await this.appsApi.readNamespacedDeployment({ name, namespace: ns })
      if (this.isHccOwned(dep, podKey)) {
        await this.appsApi.deleteNamespacedDeployment({ name, namespace: ns })
        hccLogger.info('Deployment deleted (0 members)', { scope: LOG, deployment: name })
      } else {
        hccLogger.warn('Skipping Deployment delete — not HCC-owned', {
          scope: LOG,
          deployment: name,
        })
      }
    } catch (error) {
      if (getErrorCode(error) !== 404) throw error
    }
    // Service
    try {
      const svc = await this.coreApi.readNamespacedService({ name, namespace: ns })
      if (this.isHccOwned(svc, podKey)) {
        await this.coreApi.deleteNamespacedService({ name, namespace: ns })
        hccLogger.info('Service deleted (0 members)', { scope: LOG, service: name })
      } else {
        hccLogger.warn('Skipping Service delete — not HCC-owned', { scope: LOG, service: name })
      }
    } catch (error) {
      if (getErrorCode(error) !== 404) throw error
    }
    // NetworkPolicy
    try {
      const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
      if (this.isHccOwned(np, podKey)) {
        await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
        hccLogger.info('NetworkPolicy deleted (0 members)', { scope: LOG, policy: name })
      } else {
        hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {
          scope: LOG,
          policy: name,
        })
      }
    } catch (error) {
      if (getErrorCode(error) !== 404) throw error
    }
  }

  /**
   * Startup / periodic orphan GC (§3): list all `llmhook-*` Deployments by
   * label and delete any whose hook-pod-key has zero live members — covers pods
   * orphaned by a missed delete event or a crash between CR-delete and
   * workload-delete. Mandatory (no owner-ref cascade to rely on).
   */
  private async sweepOrphanedWorkloads(): Promise<void> {
    let deployments: k8s.V1Deployment[]
    try {
      const resp = await this.appsApi.listNamespacedDeployment({
        namespace: config.llmHooksNamespace,
        labelSelector: `${MANAGED_BY_LABEL}=${MANAGED_BY_VALUE}`,
      })
      deployments = resp.items ?? []
    } catch (error) {
      hccLogger.error('Orphan sweep failed to list Deployments', { scope: LOG, err: error })
      return
    }
    for (const dep of deployments) {
      const podKey = dep.metadata?.labels?.[HOOK_PODKEY_LABEL]
      if (!podKey) continue
      if (this.membersForPodKey(podKey).length > 0) continue
      hccLogger.info('Orphan sweep deleting workload with 0 members', { scope: LOG, podKey })
      await this.runSerialized(podKey, () => this.gcPodKey(podKey))
    }
  }

  // ─── Readiness + observedDigest ─────────────────────────────────────

  private async readDeploymentRollout(
    name: string
  ): Promise<{ ready: boolean; readyReplicas: number; detail: string }> {
    try {
      const dep = await this.appsApi.readNamespacedDeployment({
        name,
        namespace: config.llmHooksNamespace,
      })
      const desired = dep.spec?.replicas ?? 1
      const readyReplicas = dep.status?.readyReplicas ?? 0
      return {
        ready: readyReplicas >= desired,
        readyReplicas,
        detail: `ready ${readyReplicas}/${desired}`,
      }
    } catch (error) {
      return { ready: false, readyReplicas: 0, detail: `Deployment read failed: ${errMsg(error)}` }
    }
  }

  /**
   * observedDigest from the LIVE pod's container status (§6). Reflects the
   * digest ACTUALLY running (not merely the intended one), which is what
   * mcp-host binds Host.spec.guardrails.hooks[].digest against and fails closed
   * on. Needs `pods` get/list RBAC in llm-hooks. Empty string when unresolved.
   */
  private async readObservedDigest(podKey: string): Promise<string> {
    try {
      const pods = await this.coreApi.listNamespacedPod({
        namespace: config.llmHooksNamespace,
        labelSelector: `${HOOK_PODKEY_LABEL}=${podKey}`,
      })
      for (const pod of pods.items ?? []) {
        for (const cs of pod.status?.containerStatuses ?? []) {
          const digest = extractSha256(cs.imageID) ?? extractSha256(cs.image)
          if (digest) return digest
        }
      }
    } catch (error) {
      hccLogger.warn('Failed to read observedDigest', { scope: LOG, podKey, err: error })
    }
    return ''
  }

  // ─── Status subresource writer (mirror writeStatusCondition) ──────────

  /**
   * Write (merge) the `Ready` condition plus flat observedDigest/readyReplicas/
   * lastReconciled on an LlmHook, fenced by a resourceVersion `test` op with
   * retry-on-409/422 so a concurrent writer never clobbers the other's fields.
   * Short-circuits when nothing changed (prevents a status→watch→reconcile loop).
   */
  private async writeStatus(
    hook: LlmHookCRD,
    condition: Omit<LlmHookCondition, 'lastTransitionTime'>,
    extras: StatusExtras = {}
  ): Promise<void> {
    const MAX_ATTEMPTS = 4
    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      const now = new Date().toISOString()
      let current: { metadata?: { resourceVersion?: string }; status?: LlmHookStatus }
      try {
        current = (await this.customApi.getNamespacedCustomObjectStatus({
          group: GROUP,
          version: VERSION,
          namespace: hook.namespace,
          plural: PLURAL_LLMHOOKS,
          name: hook.name,
        })) as { metadata?: { resourceVersion?: string }; status?: LlmHookStatus }
      } catch (error) {
        if (getErrorCode(error) === 404) return
        hccLogger.warn('Failed to read LlmHook status', { scope: LOG, hook: hook.name, err: error })
        return
      }

      const resourceVersion = current.metadata?.resourceVersion
      const hasStatusObject = typeof current.status === 'object' && current.status !== null
      const existing = current.status ?? {}
      const existingConditions = existing.conditions ?? []

      const prior = existingConditions.find(c => c.type === condition.type)
      const lastTransitionTime =
        prior && prior.status === condition.status ? (prior.lastTransitionTime ?? now) : now
      const merged: LlmHookCondition = {
        type: condition.type,
        status: condition.status,
        reason: condition.reason,
        message: condition.message,
        lastTransitionTime,
        ...(hook.generation !== undefined && { observedGeneration: hook.generation }),
      }
      const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]

      const nextStatus: LlmHookStatus = {
        ...existing,
        conditions: nextConditions,
        ...(extras.observedDigest !== undefined && { observedDigest: extras.observedDigest }),
        ...(extras.readyReplicas !== undefined && { readyReplicas: extras.readyReplicas }),
        ...(extras.lastReconciled !== undefined && { lastReconciled: extras.lastReconciled }),
      }

      // No-op gate: skip the patch when nothing meaningful changed, so a status
      // write does not self-trigger an endless reconcile loop.
      const conditionUnchanged =
        prior !== undefined &&
        prior.status === condition.status &&
        prior.reason === condition.reason &&
        prior.message === condition.message
      const digestUnchanged =
        extras.observedDigest === undefined || existing.observedDigest === extras.observedDigest
      const replicasUnchanged =
        extras.readyReplicas === undefined || existing.readyReplicas === extras.readyReplicas
      if (conditionUnchanged && digestUnchanged && replicasUnchanged) return

      const statusPatch = [
        ...(resourceVersion
          ? [{ op: 'test', path: '/metadata/resourceVersion', value: resourceVersion }]
          : []),
        hasStatusObject
          ? { op: 'add', path: '/status/conditions', value: nextConditions }
          : { op: 'add', path: '/status', value: nextStatus },
        ...(hasStatusObject && extras.observedDigest !== undefined
          ? [{ op: 'add', path: '/status/observedDigest', value: extras.observedDigest }]
          : []),
        ...(hasStatusObject && extras.readyReplicas !== undefined
          ? [{ op: 'add', path: '/status/readyReplicas', value: extras.readyReplicas }]
          : []),
        ...(hasStatusObject && extras.lastReconciled !== undefined
          ? [{ op: 'add', path: '/status/lastReconciled', value: extras.lastReconciled }]
          : []),
      ]

      try {
        await this.customApi.patchNamespacedCustomObjectStatus({
          group: GROUP,
          version: VERSION,
          namespace: hook.namespace,
          plural: PLURAL_LLMHOOKS,
          name: hook.name,
          body: statusPatch,
        })
        return
      } catch (error) {
        if (getErrorCode(error) === 404) return
        const code = getErrorCode(error)
        if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) continue
        hccLogger.warn('Failed to write LlmHook status', {
          scope: LOG,
          hook: hook.name,
          err: error,
        })
        return
      }
    }
  }
}

function condition(
  status: 'True' | 'False' | 'Unknown',
  reason: LlmHookReadyReason,
  message: string
): Omit<LlmHookCondition, 'lastTransitionTime'> {
  return { type: 'Ready', status, reason, message }
}

function extractSha256(ref: string | undefined): string | undefined {
  if (!ref) return undefined
  const m = ref.match(/sha256:[a-f0-9]{64}/)
  return m ? m[0] : undefined
}

function errMsg(error: unknown): string {
  return error instanceof Error ? error.message : String(error)
}