Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
39.3 KB
Raw Blame
import * as k8s from '@kubernetes/client-node'
import { rootLogger } from '../observability/logger.js'
import { governedTraceAdministrativeIntentReconcileFailedTotal } from '../observability/metrics.js'
import {
  CLERUM_GROUP,
  CLERUM_VERSION,
  ClerumResource,
  ClerumResourceType,
  ResourceListResponse,
  ResourcePreconditions,
} from '../types.js'
import {
  addNonEmpty,
  extractK8sStatus,
  kindFromPlural,
  parseProjectedGeneration,
} from './resourceServiceHelpers.js'
import {
  ADMINISTRATIVE_INTENT_ANNOTATION,
  ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION,
} from './tracing/adminOperationConstants.js'
import { currentAdministrativeRequestContext } from './tracing/adminOperationContext.js'
import {
  ControlApiAdministrativeOperationService,
  type HostAdministrativeAction,
  type HostAdministrativeIntent,
  stripAdministrativeIntentAnnotation,
  withAdministrativeIntentAnnotation,
} from './tracing/adminOperationService.js'

let administrativeOperationService: ControlApiAdministrativeOperationService | null = null

export function setAdministrativeOperationService(
  service: ControlApiAdministrativeOperationService | null
): void {
  administrativeOperationService = service
}

export class K8sNotFoundError extends Error {
  readonly httpStatus = 404
  constructor(message: string) {
    super(message)
    this.name = 'K8sNotFoundError'
  }
}

/**
 * AP-6 (docs/architecture/stateless-invariants.md): the caller supplied the
 * resourceVersion it READ and the object changed since that read. Retrying
 * would re-apply the same stale payload over the concurrent write, so the
 * conflict is surfaced to the caller instead of being retried away.
 */
export class K8sConflictError extends Error {
  readonly httpStatus = 409
  constructor(message: string) {
    super(message)
    this.name = 'K8sConflictError'
  }
}

export type MutableResourceSnapshot = {
  metadata?: {
    annotations?: Record<string, string>
    labels?: Record<string, string>
    uid?: string
    resourceVersion?: string
    generation?: number
    ownerReferences?: Array<Record<string, unknown>>
    finalizers?: string[]
  }
  spec?: Record<string, unknown>
}

type ResourceMutation = {
  metadata?: { labels?: Record<string, string>; annotations?: Record<string, string> }
  spec: Record<string, unknown>
} | null

/** Annotation keys under this prefix are platform-owned write-only projections. */
const PLATFORM_ANNOTATION_PREFIX = 'clerum.io/'

function isHostResource(plural: ClerumResourceType): boolean {
  return plural === 'hosts'
}

async function persistHostIntent(input: {
  plural: ClerumResourceType
  action: HostAdministrativeAction
  namespace: string
  name: string
}): Promise<HostAdministrativeIntent | null> {
  if (!isHostResource(input.plural) || !administrativeOperationService) return null
  const requestContext = currentAdministrativeRequestContext()
  if (!requestContext) return null
  return administrativeOperationService.persistHostIntent({
    action: input.action,
    namespace: input.namespace,
    name: input.name,
    operatorSub: requestContext.operatorSub,
    requestId: requestContext.requestId,
  })
}

async function persistHostFailure(intent: HostAdministrativeIntent | null): Promise<void> {
  if (!intent || !administrativeOperationService) return
  await administrativeOperationService.persistHostOutcome(
    intent,
    'failed',
    'kubernetes_write_failed'
  )
}

/**
 * Merge annotations for a full-replace write (updateResource/mutateResource).
 *
 * - `incoming === undefined` — the caller did not touch annotations: the
 *   server's current map survives verbatim (legacy behavior).
 * - `incoming` provided — caller-owned keys keep replace semantics (the
 *   incoming map wins per key, omission clears), but platform-owned
 *   `clerum.io/*` keys present on the server are re-added unless the caller
 *   EXPLICITLY sets that exact key. Platform annotations are write-only
 *   projections owned by control-api/HCC (e.g. `clerum.io/wake-requested`,
 *   see hostWakeService.ts) — an admin PUT that happens to carry an
 *   annotations map must never erase them (AP-6 companion fix).
 */
export function mergeAnnotationsForReplace(
  current: Record<string, string> | undefined,
  incoming: Record<string, string> | undefined
): Record<string, string> | undefined {
  if (incoming === undefined) return current
  const merged: Record<string, string> = { ...incoming }
  for (const [key, value] of Object.entries(current ?? {})) {
    if (key.startsWith(PLATFORM_ANNOTATION_PREFIX) && !(key in incoming)) {
      merged[key] = value
    }
  }
  return merged
}

/** Preserve the kubeconfig client's server, auth and TLS middleware per call. */
function cancellationOptions(signal?: AbortSignal): k8s.ConfigurationOptions | undefined {
  signal?.throwIfAborted()
  if (!signal) return undefined
  const cancellation = k8s.createConfiguration({
    promiseMiddleware: [
      {
        pre: async context => {
          signal.throwIfAborted()
          context.setSignal(signal)
          return context
        },
        post: async response => response,
      },
    ],
  })
  return { middleware: cancellation.middleware, middlewareMergeStrategy: 'append' }
}

export class ResourceService {
  // Allowed namespaces derived from config at construction time.
  private readonly allowedNamespaces: Set<string>

  constructor(
    private readonly customApi: k8s.CustomObjectsApi,
    private readonly defaultNamespace: string,
    // Supports string (single namespace) or string[] (multiple namespaces for list operations).
    // The first element of an array is used as the primary namespace for create/get/update/delete.
    private readonly defaultNamespaces: Partial<Record<ClerumResourceType, string | string[]>>
  ) {
    // Build allowed set from all configured namespaces + the default
    const nsSet = new Set<string>([this.defaultNamespace])
    for (const ns of Object.values(this.defaultNamespaces)) {
      addNonEmpty(nsSet, ns)
    }
    this.allowedNamespaces = nsSet
  }

  private resolveNamespace(plural: ClerumResourceType, namespace?: string): string {
    if (namespace && namespace.trim()) {
      const ns = namespace.trim()
      if (!this.allowedNamespaces.has(ns)) {
        throw new Error(`Namespace not allowed: ${ns}`)
      }
      return ns
    }
    const configured = this.defaultNamespaces[plural]
    // For arrays, the first entry is the primary (creation) namespace.
    const primary = Array.isArray(configured) ? configured[0] : configured
    return primary || this.defaultNamespace
  }

  private listNamespacesForPlural(plural: ClerumResourceType): string[] {
    const namespaces = new Set<string>()
    addNonEmpty(namespaces, this.defaultNamespaces[plural])
    addNonEmpty(namespaces, this.defaultNamespace)
    return Array.from(namespaces)
  }

  async listResource(
    plural: ClerumResourceType,
    namespace?: string,
    signal?: AbortSignal
  ): Promise<unknown[]> {
    signal?.throwIfAborted()
    if (namespace === '*') {
      const items: unknown[] = []
      for (const nsName of this.listNamespacesForPlural(plural)) {
        try {
          const namespaced = (await this.customApi.listNamespacedCustomObject(
            {
              group: CLERUM_GROUP,
              version: CLERUM_VERSION,
              namespace: nsName,
              plural,
            },
            cancellationOptions(signal)
          )) as unknown as ResourceListResponse
          signal?.throwIfAborted()
          items.push(...(namespaced.items || []))
        } catch {
          signal?.throwIfAborted()
          // Ignore namespaces where the resource is absent/inaccessible.
        }
      }
      return items
    }

    const resolvedNamespace = this.resolveNamespace(plural, namespace)
    const res = (await this.customApi.listNamespacedCustomObject(
      {
        group: CLERUM_GROUP,
        version: CLERUM_VERSION,
        namespace: resolvedNamespace,
        plural,
      },
      cancellationOptions(signal)
    )) as unknown as ResourceListResponse
    signal?.throwIfAborted()
    return res.items || []
  }

  async getResource(
    plural: ClerumResourceType,
    name: string,
    namespace?: string,
    signal?: AbortSignal
  ): Promise<unknown> {
    signal?.throwIfAborted()
    const ns = this.resolveNamespace(plural, namespace)
    try {
      const resource = await this.customApi.getNamespacedCustomObject(
        {
          group: CLERUM_GROUP,
          version: CLERUM_VERSION,
          namespace: ns,
          plural,
          name,
        },
        cancellationOptions(signal)
      )
      signal?.throwIfAborted()
      return resource
    } catch (err) {
      // node-fetch reports AbortError; retain the owner's typed reason instead.
      signal?.throwIfAborted()
      const status = extractK8sStatus(err)
      if (namespace && namespace.trim()) {
        if (status === 404) {
          throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
        }
        throw err
      }
      if (status !== 404 && status !== null) {
        throw err
      }
      const items = (await this.listResource(plural, '*', signal)) as Array<{
        metadata?: { name?: string }
      }>
      const match = items.find(i => i.metadata?.name === name)
      if (!match)
        throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns} or cluster-wide`)
      return match
    }
  }

  async createResource(
    plural: ClerumResourceType,
    body: {
      metadata: {
        name: string
        labels?: Record<string, string>
        annotations?: Record<string, string>
      }
      spec: Record<string, unknown>
    },
    namespace?: string
  ): Promise<unknown> {
    // Namespaces are determined by the HTTP layer from server-side config; the
    // service intentionally ignores any namespace the body might carry.
    const ns = this.resolveNamespace(plural, namespace)
    const intent = await persistHostIntent({
      plural,
      action: 'create',
      namespace: ns,
      name: body.metadata.name,
    })
    const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata) ?? body.metadata
    // A create is the one prediction that cannot be wrong: the apiserver's
    // `PrepareForCreate` sets generation 1 unconditionally, and it runs AFTER
    // mutating admission, so no webhook can land the object anywhere else. The
    // value still goes through `reconcileAdministrativeIntentGeneration` below,
    // which returns without patching once it confirms 1 === 1 — the check is
    // what makes the invariant observable instead of assumed.
    const predictedGeneration = 1
    const annotations = intent
      ? withAdministrativeIntentAnnotation(
          sanitizedMetadata.annotations,
          intent,
          predictedGeneration
        )
      : sanitizedMetadata.annotations
    const resource: ClerumResource = {
      apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
      kind: kindFromPlural(plural),
      metadata: {
        name: body.metadata.name,
        namespace: ns,
        ...(sanitizedMetadata.labels && { labels: sanitizedMetadata.labels }),
        ...(annotations && { annotations }),
      },
      spec: body.spec,
    }

    let created: unknown
    try {
      created = await this.customApi.createNamespacedCustomObject({
        group: CLERUM_GROUP,
        version: CLERUM_VERSION,
        namespace: ns,
        plural,
        body: resource,
      })
    } catch (err) {
      await persistHostFailure(intent)
      throw err
    }
    // Outside the try on purpose: the create already succeeded, and a failure
    // to correct the annotation must not reach `persistHostFailure`, which
    // would record a completed operation as failed.
    await this.reconcileAdministrativeIntentGeneration({
      plural,
      name: body.metadata.name,
      namespace: ns,
      intent,
      predictedGeneration,
      written: created,
    })
    return created
  }

  async updateResource(
    plural: ClerumResourceType,
    name: string,
    body: {
      metadata?: {
        labels?: Record<string, string>
        annotations?: Record<string, string>
        /**
         * AP-6 identity precondition. When present, the replacement must
         * still address the same Kubernetes object the caller read. This is
         * required for compensations: a same-name delete/recreate must never
         * receive the old object's rollback payload.
         */
        uid?: string
        /**
         * AP-6 — reader's-version precondition. When present, this is the
         * resourceVersion the CALLER READ (e.g. the version the control-ui
         * edit form was built from). It is used as the replace precondition
         * INSTEAD of the server's current version, and a 409 surfaces as
         * K8sConflictError without retrying: re-reading only to re-apply the
         * same stale payload is the lost-update bug optimistic concurrency
         * exists to prevent. When absent, legacy last-write-wins behavior is
         * preserved (server-version precondition + bounded retry) for
         * API-only callers that intentionally replace whatever is current.
         */
        resourceVersion?: string
      }
      spec: Record<string, unknown>
    },
    namespace?: string,
    options?: {
      /**
       * N1 — a CR snapshot the caller ALREADY read for this same object (e.g.
       * the admin PUT ratchet read in resources.ts). When present it is reused
       * for the FIRST replace attempt instead of issuing a second apiserver
       * GET, collapsing the per-PUT read count from 2 to 1. On a 409 retry the
       * loop falls back to a fresh read (the pre-read is stale by definition),
       * so the last-write-wins semantics are unchanged. Never used past
       * attempt 1.
       */
      preReadCurrent?: MutableResourceSnapshot
    }
  ): Promise<unknown> {
    const ns = this.resolveNamespace(plural, namespace)
    const intent = await persistHostIntent({ plural, action: 'update', namespace: ns, name })
    const readerUid = body.metadata?.uid || undefined
    const readerResourceVersion = body.metadata?.resourceVersion || undefined
    // A reader resourceVersion freezes the caller's edit and must not be retried:
    // re-reading then replaying that stale payload would lose a concurrent update.
    // A UID alone guards only against same-name delete/recreate, so it keeps the
    // legacy bounded retry contract while every fresh read revalidates identity.
    const maxAttempts = readerResourceVersion ? 1 : 3
    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
      const current = (
        attempt === 1 && options?.preReadCurrent
          ? options.preReadCurrent
          : await this.getResource(plural, name, ns)
      ) as {
        metadata?: {
          annotations?: Record<string, string>
          labels?: Record<string, string>
          uid?: string
          resourceVersion?: string
          generation?: number
          ownerReferences?: Array<Record<string, unknown>>
          finalizers?: string[]
        }
      }

      // resourceVersion detects a stale edit; UID detects a delete/recreate
      // under the same name. Both are checked before constructing the replace
      // body so an identity-bound compensation fails closed without sending
      // any payload to the apiserver.
      if (readerUid && current.metadata?.uid !== readerUid) {
        await persistHostFailure(intent)
        throw new K8sConflictError(
          `${plural}/${name} changed identity since it was read (stale uid ${readerUid})`
        )
      }

      const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata)
      const mergedAnnotations = mergeAnnotationsForReplace(
        current.metadata?.annotations,
        sanitizedMetadata?.annotations
      )
      // Predicted, not persisted: the annotation travels inside the same body
      // as the spec, so the generation the apiserver will assign is not
      // knowable yet. `reconcileAdministrativeIntentGeneration` corrects it
      // against the response — a replace whose spec is unchanged leaves the
      // generation where it was, and the prediction would pin the intent to a
      // generation that never arrives.
      const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
      const annotations = intent
        ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
        : mergedAnnotations
      const resource: ClerumResource = {
        apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
        kind: kindFromPlural(plural),
        metadata: {
          name,
          namespace: ns,
          ...(readerUid ? { uid: readerUid } : {}),
          ...(current.metadata?.labels && { labels: current.metadata.labels }),
          ...(sanitizedMetadata?.labels && { labels: sanitizedMetadata.labels }),
          ...(annotations && { annotations }),
          ...(current.metadata?.ownerReferences && {
            ownerReferences: current.metadata.ownerReferences,
          }),
          ...(current.metadata?.finalizers && { finalizers: current.metadata.finalizers }),
        },
        spec: body.spec,
      }

      let replaced: unknown
      try {
        replaced = await this.customApi.replaceNamespacedCustomObject({
          group: CLERUM_GROUP,
          version: CLERUM_VERSION,
          namespace: ns,
          plural,
          name,
          body: {
            ...resource,
            metadata: {
              ...resource.metadata,
              resourceVersion: readerResourceVersion ?? current.metadata?.resourceVersion,
            },
          },
        })
      } catch (err) {
        if (extractK8sStatus(err) === 409) {
          if (readerResourceVersion) {
            await persistHostFailure(intent)
            throw new K8sConflictError(
              `${plural}/${name} changed since it was read (stale resourceVersion ${readerResourceVersion})`
            )
          }
          if (attempt < maxAttempts) {
            continue
          }
        }
        await persistHostFailure(intent)
        throw err
      }
      // After the catch, not inside the try: the replace already succeeded, so
      // a failure here must not reach `persistHostFailure` and record a
      // completed operation as failed.
      await this.reconcileAdministrativeIntentGeneration({
        plural,
        name,
        namespace: ns,
        intent,
        predictedGeneration,
        written: replaced,
      })
      return replaced
    }
    throw new Error(`Failed to update ${plural}/${name} after ${maxAttempts} attempts`)
  }

  /**
   * Replace the PREDICTED generation in the administrative-intent annotation
   * with the one the apiserver actually persisted (#329).
   *
   * The three callers — `createResource` (:261), `updateResource` (:375) and
   * `mutateResource` (:766) — annotate BEFORE the write, because the annotation
   * has to travel inside the same body. `createResource` predicts the literal
   * `1`; the other two predict `current.generation + 1`. That prediction is
   * wrong whenever the write does not bump the generation, and the Host CRD
   * declares a status subresource (`charts/clerum-crds/crds/host.yaml`), so
   * `metadata.generation` advances ONLY on spec changes. A replace whose spec
   * is byte-identical — an idempotent operator PUT, a `mutateResource` whose
   * mutation is a no-op, a hook-ref resync — leaves the object at N while the
   * annotation says N+1. Nothing ever retires that annotation, so the Host
   * stops binding outcomes forever.
   *
   * The correction only ever LOWERS the annotation, and that bound is what
   * lets the binding resolver treat `annotated < live` as terminal. Every
   * replace carries a `resourceVersion` precondition (:404, :795), so the
   * apiserver's `old` object is the one this service read and
   * `PrepareForUpdate` can raise the generation by at most one: `persisted` is
   * therefore never greater than `predicted`. On a create it is neither —
   * `PrepareForCreate` sets generation 1 unconditionally, after mutating
   * admission has already run, so `persisted === predicted === 1` and the
   * guard below returns before any patch is issued.
   *
   * A merge-patch of annotations alone does not bump the generation, so this
   * correction cannot feed itself.
   *
   * NOT `patchAnnotationMonotonic`: that method takes the max of projected and
   * requested and refuses to lower a value. Lowering N+1 back to N is exactly
   * the correction needed here.
   *
   * Never throws. The write it corrects already succeeded, and turning a
   * successful mutation into a failed request would misreport what happened to
   * the object. Everything it cannot do is logged with the reason.
   */
  private async reconcileAdministrativeIntentGeneration(input: {
    plural: ClerumResourceType
    name: string
    namespace: string
    intent: HostAdministrativeIntent | null
    predictedGeneration: number
    written: unknown
  }): Promise<void> {
    if (!input.intent) return
    const metadata = (input.written as { metadata?: { generation?: number } } | null)?.metadata
    const persisted = metadata?.generation
    if (typeof persisted !== 'number' || !Number.isSafeInteger(persisted)) {
      governedTraceAdministrativeIntentReconcileFailedTotal.inc({
        reason: 'generation_missing',
      })
      rootLogger.warn(
        {
          event: 'administrative_intent_generation_missing',
          plural: input.plural,
          namespace: input.namespace,
          name: input.name,
          predictedGeneration: input.predictedGeneration,
        },
        'write response carried no usable metadata.generation; the intent annotation was left at the predicted value'
      )
      return
    }
    if (persisted === input.predictedGeneration) return

    const maxAttempts = 3
    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
      // The first attempt uses the write's own response; a retry re-reads,
      // because a 409 means the resourceVersion we held is stale.
      let snapshot: MutableResourceSnapshot
      if (attempt === 1) {
        snapshot = input.written as MutableResourceSnapshot
      } else {
        try {
          snapshot = (await this.getResource(
            input.plural,
            input.name,
            input.namespace
          )) as MutableResourceSnapshot
        } catch (err) {
          // One of the two places in this method that can throw — the patch
          // below is the other, which is why it carries its own catch. Both are
          // caught so the documented contract holds, and both report the reason
          // rather than swallowing it.
          governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'reread_failed' })
          rootLogger.warn(
            {
              event: 'administrative_intent_generation_reread_failed',
              plural: input.plural,
              namespace: input.namespace,
              name: input.name,
              predictedGeneration: input.predictedGeneration,
              persistedGeneration: persisted,
              status: extractK8sStatus(err),
            },
            're-read failed while correcting the intent annotation to the persisted generation'
          )
          return
        }
      }
      // Ownership, not generation, decides whether this correction is still
      // ours to make. The two annotation keys are written as a PAIR
      // (`withAdministrativeIntentAnnotation`), so once another administrative
      // operation has landed, the id key names ITS operation and patching the
      // generation key alone would fuse its id to our generation — a pair that
      // never existed. Its own reconcile will correct its own prediction.
      const liveOperationId = snapshot.metadata?.annotations?.[ADMINISTRATIVE_INTENT_ANNOTATION]
      if (liveOperationId === undefined) {
        // Absent is not reassigned, and the two must not share a log line. No
        // id key means no obligation is pending on this object at all, so
        // writing the generation key alone would create half a pair that names
        // no operation — worse than the prediction it replaces.
        governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'intent_absent' })
        rootLogger.warn(
          {
            event: 'administrative_intent_generation_intent_absent',
            plural: input.plural,
            namespace: input.namespace,
            name: input.name,
            predictedGeneration: input.predictedGeneration,
            persistedGeneration: persisted,
          },
          'no administrative intent id on the object; the generation annotation was left uncorrected'
        )
        return
      }
      if (liveOperationId !== input.intent.operationId) {
        rootLogger.warn(
          {
            event: 'administrative_intent_generation_reassigned',
            plural: input.plural,
            namespace: input.namespace,
            name: input.name,
            writtenGeneration: persisted,
            liveGeneration: snapshot.metadata?.generation,
          },
          'another administrative operation owns the intent annotation; this prediction was left uncorrected'
        )
        return
      }
      const resourceVersion = snapshot.metadata?.resourceVersion
      if (!resourceVersion) {
        governedTraceAdministrativeIntentReconcileFailedTotal.inc({
          reason: 'no_precondition',
        })
        rootLogger.warn(
          {
            event: 'administrative_intent_generation_no_precondition',
            plural: input.plural,
            namespace: input.namespace,
            name: input.name,
            predictedGeneration: input.predictedGeneration,
            persistedGeneration: persisted,
          },
          'no metadata.resourceVersion to use as a patch precondition; the intent annotation was left at the predicted value'
        )
        return
      }
      if (snapshot.metadata?.generation !== persisted) {
        // A non-administrative writer bumped the spec without touching the
        // annotations, so the object is now past the generation this operation
        // produced. Correct it ANYWAY: `persisted` is the generation this write
        // actually produced, which stays true wherever the object has got to,
        // and writing it lands `annotated < live` — the terminal refusal the
        // resolver exists to make.
        //
        // Leaving the PREDICTION in place is the dangerous option, and it is
        // the one this branch used to take. `predicted` is by construction the
        // next generation the object is most likely to reach, so a concurrent
        // bump can make it match the live generation exactly, and then the
        // resolver binds another writer's change to this operator's intent —
        // the audit-trail falsification #329 exists to prevent, reintroduced by
        // the code meant to prevent it.
        rootLogger.warn(
          {
            event: 'administrative_intent_generation_outpaced',
            plural: input.plural,
            namespace: input.namespace,
            name: input.name,
            writtenGeneration: persisted,
            liveGeneration: snapshot.metadata?.generation,
          },
          'object advanced past the generation this operation produced; correcting the annotation to it anyway'
        )
      }
      try {
        await this.customApi.patchNamespacedCustomObject(
          {
            group: CLERUM_GROUP,
            version: CLERUM_VERSION,
            namespace: input.namespace,
            plural: input.plural,
            name: input.name,
            body: {
              metadata: {
                resourceVersion,
                annotations: {
                  [ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION]: String(persisted),
                },
              },
            },
          },
          k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
        )
        return
      } catch (err) {
        if (extractK8sStatus(err) === 409 && attempt < maxAttempts) continue
        governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'patch_failed' })
        rootLogger.warn(
          {
            event: 'administrative_intent_generation_patch_failed',
            plural: input.plural,
            namespace: input.namespace,
            name: input.name,
            predictedGeneration: input.predictedGeneration,
            persistedGeneration: persisted,
            status: extractK8sStatus(err),
          },
          'could not correct the intent annotation to the persisted generation'
        )
        return
      }
    }
  }

  /**
   * Merge-patch ONLY metadata.annotations of a custom object.
   *
   * Unlike updateResource/mutateResource this is a single server-side JSON
   * merge patch: no read-modify-write, no resourceVersion, no retry loop —
   * concurrent full replaces (e.g. the admin facade) cannot clobber it and it
   * cannot clobber them. Used for write-only annotation projections such as
   * `clerum.io/wake-requested`.
   */
  async patchResourceAnnotations(
    plural: ClerumResourceType,
    name: string,
    annotations: Record<string, string>,
    namespace?: string
  ): Promise<unknown> {
    const ns = this.resolveNamespace(plural, namespace)
    try {
      return await this.customApi.patchNamespacedCustomObject(
        {
          group: CLERUM_GROUP,
          version: CLERUM_VERSION,
          namespace: ns,
          plural,
          name,
          body: { metadata: { annotations } },
        },
        k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
      )
    } catch (err) {
      if (extractK8sStatus(err) === 404) {
        throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
      }
      throw err
    }
  }

  /**
   * Monotonically project a single numeric annotation whose value must never
   * regress under concurrent writers.
   *
   * Why this exists (and why a blind `patchResourceAnnotations` merge patch is
   * unsafe here): the wake generation is a strictly monotonic counter in
   * Postgres (the source of truth). The annotation is only a WRITE-ONLY
   * PROJECTION of that counter for HCC to observe. A blind merge patch carries
   * no ordering guarantee — with coalesce-window = 0 or control-api replicas
   * > 1, a slower gen=5 patch can land AFTER a faster gen=6 patch and REGRESS
   * the projected annotation. HCC advances `wakeHandledGeneration` up to the
   * value it observes; a regressed projection can briefly exceed the projected
   * value and SUPPRESS a later legitimate wake event. Bounded (the host still
   * wakes via resync) but it violates the "strictly monotonic projection"
   * invariant this projection is supposed to uphold.
   *
   * Fix: max-semantics with an optimistic-concurrency precondition.
   *   - Read the current projected value. If it is already >= the new
   *     generation, do NOT patch (never regress).
   *   - Otherwise merge-patch the annotation WITH the object's
   *     `metadata.resourceVersion` as a precondition. The apiserver rejects
   *     the patch with 409 if the object changed since the read; on 409 we
   *     re-read and re-evaluate max-semantics, which also lets a concurrent
   *     higher-generation write win the race.
   * The generation itself always comes from Postgres — this method only
   * decides WHETHER (and with which precondition) to project it.
   */
  async patchAnnotationMonotonic(
    plural: ClerumResourceType,
    name: string,
    annotationKey: string,
    generation: number,
    namespace?: string
  ): Promise<unknown> {
    if (!Number.isFinite(generation) || generation < 0 || !Number.isInteger(generation)) {
      throw new Error(
        `patchAnnotationMonotonic: generation must be a non-negative integer, got ${generation}`
      )
    }
    const ns = this.resolveNamespace(plural, namespace)
    const maxAttempts = 5
    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
      const current = (await this.getResource(plural, name, ns)) as {
        metadata?: {
          annotations?: Record<string, string>
          resourceVersion?: string
        }
      }
      const projected = parseProjectedGeneration(current.metadata?.annotations?.[annotationKey])
      if (projected !== null && projected >= generation) {
        // The annotation already projects an equal-or-higher generation.
        // Patching would regress (lower value) or be a no-op (equal value):
        // in both cases skip, preserving the monotonic invariant.
        return current
      }

      const resourceVersion = current.metadata?.resourceVersion
      if (!resourceVersion) {
        // Fail loud: without a resourceVersion we cannot enforce the
        // optimistic-concurrency precondition, so a blind patch could regress
        // the projection — exactly the bug this method exists to prevent.
        throw new Error(
          `patchAnnotationMonotonic: ${plural}/${name} has no metadata.resourceVersion; cannot project ${annotationKey} safely`
        )
      }

      try {
        // metadata.resourceVersion in a merge-patch body is honored by the
        // apiserver as an optimistic-concurrency precondition: the patch is
        // rejected with 409 if the object changed since the read above.
        return await this.customApi.patchNamespacedCustomObject(
          {
            group: CLERUM_GROUP,
            version: CLERUM_VERSION,
            namespace: ns,
            plural,
            name,
            body: {
              metadata: {
                resourceVersion,
                annotations: { [annotationKey]: String(generation) },
              },
            },
          },
          k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
        )
      } catch (err) {
        const status = extractK8sStatus(err)
        if (status === 404) {
          throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
        }
        if (status === 409 && attempt < maxAttempts) {
          // The object changed under us (concurrent projection or unrelated
          // metadata write). Re-read and re-evaluate max-semantics.
          continue
        }
        throw err
      }
    }
    throw new Error(
      `patchAnnotationMonotonic: failed to project ${annotationKey}=${generation} onto ${plural}/${name} after ${maxAttempts} attempts`
    )
  }

  async mutateResource(
    plural: ClerumResourceType,
    name: string,
    mutate: (current: MutableResourceSnapshot) => ResourceMutation | Promise<ResourceMutation>,
    namespace?: string
  ): Promise<unknown> {
    const ns = this.resolveNamespace(plural, namespace)
    let intent: HostAdministrativeIntent | null = null
    const maxAttempts = 3
    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
      const current = (await this.getResource(plural, name, ns)) as MutableResourceSnapshot
      const next = await mutate(current)
      if (!next) {
        return current
      }
      if (!intent) {
        intent = await persistHostIntent({ plural, action: 'config', namespace: ns, name })
      }

      const sanitizedMetadata = stripAdministrativeIntentAnnotation(next.metadata)
      // Same per-key merge as updateResource: platform-owned clerum.io/*
      // annotations survive unless the mutation explicitly sets that key.
      const mergedAnnotations = mergeAnnotationsForReplace(
        current.metadata?.annotations,
        sanitizedMetadata?.annotations
      )
      // Predicted, not persisted. This path is the likeliest producer of drift
      // in the repository: it does not short-circuit on an unchanged spec (the
      // `if (!next)` above only fires when the mutation returns nothing), so a
      // mutation that computes the same spec still issues a replace, and the
      // Host CRD's status subresource means such a replace leaves the
      // generation untouched while the prediction moves ahead of it.
      const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
      const annotations = intent
        ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
        : mergedAnnotations
      const resource: ClerumResource = {
        apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
        kind: kindFromPlural(plural),
        metadata: {
          name,
          namespace: ns,
          ...(current.metadata?.labels && { labels: current.metadata.labels }),
          ...(sanitizedMetadata?.labels && { labels: sanitizedMetadata.labels }),
          ...(annotations && { annotations }),
          ...(current.metadata?.ownerReferences && {
            ownerReferences: current.metadata.ownerReferences,
          }),
          ...(current.metadata?.finalizers && { finalizers: current.metadata.finalizers }),
        },
        spec: next.spec,
      }

      let replaced: unknown
      try {
        replaced = await this.customApi.replaceNamespacedCustomObject({
          group: CLERUM_GROUP,
          version: CLERUM_VERSION,
          namespace: ns,
          plural,
          name,
          body: {
            ...resource,
            metadata: {
              ...resource.metadata,
              resourceVersion: current.metadata?.resourceVersion,
            },
          },
        })
      } catch (err) {
        if (extractK8sStatus(err) === 409 && attempt < maxAttempts) {
          continue
        }
        await persistHostFailure(intent)
        throw err
      }
      await this.reconcileAdministrativeIntentGeneration({
        plural,
        name,
        namespace: ns,
        intent,
        predictedGeneration,
        written: replaced,
      })
      return replaced
    }
    throw new Error(`Failed to mutate ${plural}/${name} after ${maxAttempts} attempts`)
  }

  async deleteResource(
    plural: ClerumResourceType,
    name: string,
    namespace?: string,
    precondition?: ResourcePreconditions
  ): Promise<unknown> {
    const ns = this.resolveNamespace(plural, namespace)
    const intent = await persistHostIntent({ plural, action: 'delete', namespace: ns, name })
    try {
      const deleted = await this.customApi.deleteNamespacedCustomObject({
        group: CLERUM_GROUP,
        version: CLERUM_VERSION,
        namespace: ns,
        plural,
        name,
        ...(precondition && (precondition.uid || precondition.resourceVersion)
          ? {
              body: {
                preconditions: {
                  ...(precondition.uid ? { uid: precondition.uid } : {}),
                  ...(precondition.resourceVersion
                    ? { resourceVersion: precondition.resourceVersion }
                    : {}),
                },
              },
            }
          : {}),
      })
      if (intent && administrativeOperationService) {
        await administrativeOperationService.persistHostOutcome(intent, 'succeeded', 'deleted')
      }
      return deleted
    } catch (err) {
      await persistHostFailure(intent)
      throw err
    }
  }

  // Merge-patches a CRD's /status subresource. Used by the recipe retry
  // endpoint to drive the state-machine transition out of the terminal
  // `failed` phase without going through a full spec update.
  async patchResourceStatus(
    plural: ClerumResourceType,
    name: string,
    statusPatch: Record<string, unknown>,
    namespace?: string
  ): Promise<unknown> {
    const ns = this.resolveNamespace(plural, namespace)
    return this.customApi.patchNamespacedCustomObjectStatus(
      {
        group: CLERUM_GROUP,
        version: CLERUM_VERSION,
        namespace: ns,
        plural,
        name,
        body: { status: statusPatch },
      },
      { middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
    )
  }
}