Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import * as k8s from '@kubernetes/client-node'
import { rootLogger } from '../observability/logger.js'
import { governedTraceAdministrativeIntentReconcileFailedTotal } from '../observability/metrics.js'
import {
CLERUM_GROUP,
CLERUM_VERSION,
ClerumResource,
ClerumResourceType,
ResourceListResponse,
ResourcePreconditions,
} from '../types.js'
import {
addNonEmpty,
extractK8sStatus,
kindFromPlural,
parseProjectedGeneration,
} from './resourceServiceHelpers.js'
import {
ADMINISTRATIVE_INTENT_ANNOTATION,
ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION,
} from './tracing/adminOperationConstants.js'
import { currentAdministrativeRequestContext } from './tracing/adminOperationContext.js'
import {
ControlApiAdministrativeOperationService,
type HostAdministrativeAction,
type HostAdministrativeIntent,
stripAdministrativeIntentAnnotation,
withAdministrativeIntentAnnotation,
} from './tracing/adminOperationService.js'
let administrativeOperationService: ControlApiAdministrativeOperationService | null = null
export function setAdministrativeOperationService(
service: ControlApiAdministrativeOperationService | null
): void {
administrativeOperationService = service
}
export class K8sNotFoundError extends Error {
readonly httpStatus = 404
constructor(message: string) {
super(message)
this.name = 'K8sNotFoundError'
}
}
/**
* AP-6 (docs/architecture/stateless-invariants.md): the caller supplied the
* resourceVersion it READ and the object changed since that read. Retrying
* would re-apply the same stale payload over the concurrent write, so the
* conflict is surfaced to the caller instead of being retried away.
*/
export class K8sConflictError extends Error {
readonly httpStatus = 409
constructor(message: string) {
super(message)
this.name = 'K8sConflictError'
}
}
export type MutableResourceSnapshot = {
metadata?: {
annotations?: Record<string, string>
labels?: Record<string, string>
uid?: string
resourceVersion?: string
generation?: number
ownerReferences?: Array<Record<string, unknown>>
finalizers?: string[]
}
spec?: Record<string, unknown>
}
type ResourceMutation = {
metadata?: { labels?: Record<string, string>; annotations?: Record<string, string> }
spec: Record<string, unknown>
} | null
/** Annotation keys under this prefix are platform-owned write-only projections. */
const PLATFORM_ANNOTATION_PREFIX = 'clerum.io/'
function isHostResource(plural: ClerumResourceType): boolean {
return plural === 'hosts'
}
async function persistHostIntent(input: {
plural: ClerumResourceType
action: HostAdministrativeAction
namespace: string
name: string
}): Promise<HostAdministrativeIntent | null> {
if (!isHostResource(input.plural) || !administrativeOperationService) return null
const requestContext = currentAdministrativeRequestContext()
if (!requestContext) return null
return administrativeOperationService.persistHostIntent({
action: input.action,
namespace: input.namespace,
name: input.name,
operatorSub: requestContext.operatorSub,
requestId: requestContext.requestId,
})
}
async function persistHostFailure(intent: HostAdministrativeIntent | null): Promise<void> {
if (!intent || !administrativeOperationService) return
await administrativeOperationService.persistHostOutcome(
intent,
'failed',
'kubernetes_write_failed'
)
}
/**
* Merge annotations for a full-replace write (updateResource/mutateResource).
*
* - `incoming === undefined` — the caller did not touch annotations: the
* server's current map survives verbatim (legacy behavior).
* - `incoming` provided — caller-owned keys keep replace semantics (the
* incoming map wins per key, omission clears), but platform-owned
* `clerum.io/*` keys present on the server are re-added unless the caller
* EXPLICITLY sets that exact key. Platform annotations are write-only
* projections owned by control-api/HCC (e.g. `clerum.io/wake-requested`,
* see hostWakeService.ts) — an admin PUT that happens to carry an
* annotations map must never erase them (AP-6 companion fix).
*/
export function mergeAnnotationsForReplace(
current: Record<string, string> | undefined,
incoming: Record<string, string> | undefined
): Record<string, string> | undefined {
if (incoming === undefined) return current
const merged: Record<string, string> = { ...incoming }
for (const [key, value] of Object.entries(current ?? {})) {
if (key.startsWith(PLATFORM_ANNOTATION_PREFIX) && !(key in incoming)) {
merged[key] = value
}
}
return merged
}
/** Preserve the kubeconfig client's server, auth and TLS middleware per call. */
function cancellationOptions(signal?: AbortSignal): k8s.ConfigurationOptions | undefined {
signal?.throwIfAborted()
if (!signal) return undefined
const cancellation = k8s.createConfiguration({
promiseMiddleware: [
{
pre: async context => {
signal.throwIfAborted()
context.setSignal(signal)
return context
},
post: async response => response,
},
],
})
return { middleware: cancellation.middleware, middlewareMergeStrategy: 'append' }
}
export class ResourceService {
// Allowed namespaces derived from config at construction time.
private readonly allowedNamespaces: Set<string>
constructor(
private readonly customApi: k8s.CustomObjectsApi,
private readonly defaultNamespace: string,
// Supports string (single namespace) or string[] (multiple namespaces for list operations).
// The first element of an array is used as the primary namespace for create/get/update/delete.
private readonly defaultNamespaces: Partial<Record<ClerumResourceType, string | string[]>>
) {
// Build allowed set from all configured namespaces + the default
const nsSet = new Set<string>([this.defaultNamespace])
for (const ns of Object.values(this.defaultNamespaces)) {
addNonEmpty(nsSet, ns)
}
this.allowedNamespaces = nsSet
}
private resolveNamespace(plural: ClerumResourceType, namespace?: string): string {
if (namespace && namespace.trim()) {
const ns = namespace.trim()
if (!this.allowedNamespaces.has(ns)) {
throw new Error(`Namespace not allowed: ${ns}`)
}
return ns
}
const configured = this.defaultNamespaces[plural]
// For arrays, the first entry is the primary (creation) namespace.
const primary = Array.isArray(configured) ? configured[0] : configured
return primary || this.defaultNamespace
}
private listNamespacesForPlural(plural: ClerumResourceType): string[] {
const namespaces = new Set<string>()
addNonEmpty(namespaces, this.defaultNamespaces[plural])
addNonEmpty(namespaces, this.defaultNamespace)
return Array.from(namespaces)
}
async listResource(
plural: ClerumResourceType,
namespace?: string,
signal?: AbortSignal
): Promise<unknown[]> {
signal?.throwIfAborted()
if (namespace === '*') {
const items: unknown[] = []
for (const nsName of this.listNamespacesForPlural(plural)) {
try {
const namespaced = (await this.customApi.listNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: nsName,
plural,
},
cancellationOptions(signal)
)) as unknown as ResourceListResponse
signal?.throwIfAborted()
items.push(...(namespaced.items || []))
} catch {
signal?.throwIfAborted()
// Ignore namespaces where the resource is absent/inaccessible.
}
}
return items
}
const resolvedNamespace = this.resolveNamespace(plural, namespace)
const res = (await this.customApi.listNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: resolvedNamespace,
plural,
},
cancellationOptions(signal)
)) as unknown as ResourceListResponse
signal?.throwIfAborted()
return res.items || []
}
async getResource(
plural: ClerumResourceType,
name: string,
namespace?: string,
signal?: AbortSignal
): Promise<unknown> {
signal?.throwIfAborted()
const ns = this.resolveNamespace(plural, namespace)
try {
const resource = await this.customApi.getNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
},
cancellationOptions(signal)
)
signal?.throwIfAborted()
return resource
} catch (err) {
// node-fetch reports AbortError; retain the owner's typed reason instead.
signal?.throwIfAborted()
const status = extractK8sStatus(err)
if (namespace && namespace.trim()) {
if (status === 404) {
throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
}
throw err
}
if (status !== 404 && status !== null) {
throw err
}
const items = (await this.listResource(plural, '*', signal)) as Array<{
metadata?: { name?: string }
}>
const match = items.find(i => i.metadata?.name === name)
if (!match)
throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns} or cluster-wide`)
return match
}
}
async createResource(
plural: ClerumResourceType,
body: {
metadata: {
name: string
labels?: Record<string, string>
annotations?: Record<string, string>
}
spec: Record<string, unknown>
},
namespace?: string
): Promise<unknown> {
// Namespaces are determined by the HTTP layer from server-side config; the
// service intentionally ignores any namespace the body might carry.
const ns = this.resolveNamespace(plural, namespace)
const intent = await persistHostIntent({
plural,
action: 'create',
namespace: ns,
name: body.metadata.name,
})
const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata) ?? body.metadata
// A create is the one prediction that cannot be wrong: the apiserver's
// `PrepareForCreate` sets generation 1 unconditionally, and it runs AFTER
// mutating admission, so no webhook can land the object anywhere else. The
// value still goes through `reconcileAdministrativeIntentGeneration` below,
// which returns without patching once it confirms 1 === 1 — the check is
// what makes the invariant observable instead of assumed.
const predictedGeneration = 1
const annotations = intent
? withAdministrativeIntentAnnotation(
sanitizedMetadata.annotations,
intent,
predictedGeneration
)
: sanitizedMetadata.annotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
kind: kindFromPlural(plural),
metadata: {
name: body.metadata.name,
namespace: ns,
...(sanitizedMetadata.labels && { labels: sanitizedMetadata.labels }),
...(annotations && { annotations }),
},
spec: body.spec,
}
let created: unknown
try {
created = await this.customApi.createNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
body: resource,
})
} catch (err) {
await persistHostFailure(intent)
throw err
}
// Outside the try on purpose: the create already succeeded, and a failure
// to correct the annotation must not reach `persistHostFailure`, which
// would record a completed operation as failed.
await this.reconcileAdministrativeIntentGeneration({
plural,
name: body.metadata.name,
namespace: ns,
intent,
predictedGeneration,
written: created,
})
return created
}
async updateResource(
plural: ClerumResourceType,
name: string,
body: {
metadata?: {
labels?: Record<string, string>
annotations?: Record<string, string>
/**
* AP-6 identity precondition. When present, the replacement must
* still address the same Kubernetes object the caller read. This is
* required for compensations: a same-name delete/recreate must never
* receive the old object's rollback payload.
*/
uid?: string
/**
* AP-6 — reader's-version precondition. When present, this is the
* resourceVersion the CALLER READ (e.g. the version the control-ui
* edit form was built from). It is used as the replace precondition
* INSTEAD of the server's current version, and a 409 surfaces as
* K8sConflictError without retrying: re-reading only to re-apply the
* same stale payload is the lost-update bug optimistic concurrency
* exists to prevent. When absent, legacy last-write-wins behavior is
* preserved (server-version precondition + bounded retry) for
* API-only callers that intentionally replace whatever is current.
*/
resourceVersion?: string
}
spec: Record<string, unknown>
},
namespace?: string,
options?: {
/**
* N1 — a CR snapshot the caller ALREADY read for this same object (e.g.
* the admin PUT ratchet read in resources.ts). When present it is reused
* for the FIRST replace attempt instead of issuing a second apiserver
* GET, collapsing the per-PUT read count from 2 to 1. On a 409 retry the
* loop falls back to a fresh read (the pre-read is stale by definition),
* so the last-write-wins semantics are unchanged. Never used past
* attempt 1.
*/
preReadCurrent?: MutableResourceSnapshot
}
): Promise<unknown> {
const ns = this.resolveNamespace(plural, namespace)
const intent = await persistHostIntent({ plural, action: 'update', namespace: ns, name })
const readerUid = body.metadata?.uid || undefined
const readerResourceVersion = body.metadata?.resourceVersion || undefined
// A reader resourceVersion freezes the caller's edit and must not be retried:
// re-reading then replaying that stale payload would lose a concurrent update.
// A UID alone guards only against same-name delete/recreate, so it keeps the
// legacy bounded retry contract while every fresh read revalidates identity.
const maxAttempts = readerResourceVersion ? 1 : 3
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const current = (
attempt === 1 && options?.preReadCurrent
? options.preReadCurrent
: await this.getResource(plural, name, ns)
) as {
metadata?: {
annotations?: Record<string, string>
labels?: Record<string, string>
uid?: string
resourceVersion?: string
generation?: number
ownerReferences?: Array<Record<string, unknown>>
finalizers?: string[]
}
}
// resourceVersion detects a stale edit; UID detects a delete/recreate
// under the same name. Both are checked before constructing the replace
// body so an identity-bound compensation fails closed without sending
// any payload to the apiserver.
if (readerUid && current.metadata?.uid !== readerUid) {
await persistHostFailure(intent)
throw new K8sConflictError(
`${plural}/${name} changed identity since it was read (stale uid ${readerUid})`
)
}
const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata)
const mergedAnnotations = mergeAnnotationsForReplace(
current.metadata?.annotations,
sanitizedMetadata?.annotations
)
// Predicted, not persisted: the annotation travels inside the same body
// as the spec, so the generation the apiserver will assign is not
// knowable yet. `reconcileAdministrativeIntentGeneration` corrects it
// against the response — a replace whose spec is unchanged leaves the
// generation where it was, and the prediction would pin the intent to a
// generation that never arrives.
const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
const annotations = intent
? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
: mergedAnnotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
kind: kindFromPlural(plural),
metadata: {
name,
namespace: ns,
...(readerUid ? { uid: readerUid } : {}),
...(current.metadata?.labels && { labels: current.metadata.labels }),
...(sanitizedMetadata?.labels && { labels: sanitizedMetadata.labels }),
...(annotations && { annotations }),
...(current.metadata?.ownerReferences && {
ownerReferences: current.metadata.ownerReferences,
}),
...(current.metadata?.finalizers && { finalizers: current.metadata.finalizers }),
},
spec: body.spec,
}
let replaced: unknown
try {
replaced = await this.customApi.replaceNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
body: {
...resource,
metadata: {
...resource.metadata,
resourceVersion: readerResourceVersion ?? current.metadata?.resourceVersion,
},
},
})
} catch (err) {
if (extractK8sStatus(err) === 409) {
if (readerResourceVersion) {
await persistHostFailure(intent)
throw new K8sConflictError(
`${plural}/${name} changed since it was read (stale resourceVersion ${readerResourceVersion})`
)
}
if (attempt < maxAttempts) {
continue
}
}
await persistHostFailure(intent)
throw err
}
// After the catch, not inside the try: the replace already succeeded, so
// a failure here must not reach `persistHostFailure` and record a
// completed operation as failed.
await this.reconcileAdministrativeIntentGeneration({
plural,
name,
namespace: ns,
intent,
predictedGeneration,
written: replaced,
})
return replaced
}
throw new Error(`Failed to update ${plural}/${name} after ${maxAttempts} attempts`)
}
/**
* Replace the PREDICTED generation in the administrative-intent annotation
* with the one the apiserver actually persisted (#329).
*
* The three callers — `createResource` (:261), `updateResource` (:375) and
* `mutateResource` (:766) — annotate BEFORE the write, because the annotation
* has to travel inside the same body. `createResource` predicts the literal
* `1`; the other two predict `current.generation + 1`. That prediction is
* wrong whenever the write does not bump the generation, and the Host CRD
* declares a status subresource (`charts/clerum-crds/crds/host.yaml`), so
* `metadata.generation` advances ONLY on spec changes. A replace whose spec
* is byte-identical — an idempotent operator PUT, a `mutateResource` whose
* mutation is a no-op, a hook-ref resync — leaves the object at N while the
* annotation says N+1. Nothing ever retires that annotation, so the Host
* stops binding outcomes forever.
*
* The correction only ever LOWERS the annotation, and that bound is what
* lets the binding resolver treat `annotated < live` as terminal. Every
* replace carries a `resourceVersion` precondition (:404, :795), so the
* apiserver's `old` object is the one this service read and
* `PrepareForUpdate` can raise the generation by at most one: `persisted` is
* therefore never greater than `predicted`. On a create it is neither —
* `PrepareForCreate` sets generation 1 unconditionally, after mutating
* admission has already run, so `persisted === predicted === 1` and the
* guard below returns before any patch is issued.
*
* A merge-patch of annotations alone does not bump the generation, so this
* correction cannot feed itself.
*
* NOT `patchAnnotationMonotonic`: that method takes the max of projected and
* requested and refuses to lower a value. Lowering N+1 back to N is exactly
* the correction needed here.
*
* Never throws. The write it corrects already succeeded, and turning a
* successful mutation into a failed request would misreport what happened to
* the object. Everything it cannot do is logged with the reason.
*/
private async reconcileAdministrativeIntentGeneration(input: {
plural: ClerumResourceType
name: string
namespace: string
intent: HostAdministrativeIntent | null
predictedGeneration: number
written: unknown
}): Promise<void> {
if (!input.intent) return
const metadata = (input.written as { metadata?: { generation?: number } } | null)?.metadata
const persisted = metadata?.generation
if (typeof persisted !== 'number' || !Number.isSafeInteger(persisted)) {
governedTraceAdministrativeIntentReconcileFailedTotal.inc({
reason: 'generation_missing',
})
rootLogger.warn(
{
event: 'administrative_intent_generation_missing',
plural: input.plural,
namespace: input.namespace,
name: input.name,
predictedGeneration: input.predictedGeneration,
},
'write response carried no usable metadata.generation; the intent annotation was left at the predicted value'
)
return
}
if (persisted === input.predictedGeneration) return
const maxAttempts = 3
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
// The first attempt uses the write's own response; a retry re-reads,
// because a 409 means the resourceVersion we held is stale.
let snapshot: MutableResourceSnapshot
if (attempt === 1) {
snapshot = input.written as MutableResourceSnapshot
} else {
try {
snapshot = (await this.getResource(
input.plural,
input.name,
input.namespace
)) as MutableResourceSnapshot
} catch (err) {
// One of the two places in this method that can throw — the patch
// below is the other, which is why it carries its own catch. Both are
// caught so the documented contract holds, and both report the reason
// rather than swallowing it.
governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'reread_failed' })
rootLogger.warn(
{
event: 'administrative_intent_generation_reread_failed',
plural: input.plural,
namespace: input.namespace,
name: input.name,
predictedGeneration: input.predictedGeneration,
persistedGeneration: persisted,
status: extractK8sStatus(err),
},
're-read failed while correcting the intent annotation to the persisted generation'
)
return
}
}
// Ownership, not generation, decides whether this correction is still
// ours to make. The two annotation keys are written as a PAIR
// (`withAdministrativeIntentAnnotation`), so once another administrative
// operation has landed, the id key names ITS operation and patching the
// generation key alone would fuse its id to our generation — a pair that
// never existed. Its own reconcile will correct its own prediction.
const liveOperationId = snapshot.metadata?.annotations?.[ADMINISTRATIVE_INTENT_ANNOTATION]
if (liveOperationId === undefined) {
// Absent is not reassigned, and the two must not share a log line. No
// id key means no obligation is pending on this object at all, so
// writing the generation key alone would create half a pair that names
// no operation — worse than the prediction it replaces.
governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'intent_absent' })
rootLogger.warn(
{
event: 'administrative_intent_generation_intent_absent',
plural: input.plural,
namespace: input.namespace,
name: input.name,
predictedGeneration: input.predictedGeneration,
persistedGeneration: persisted,
},
'no administrative intent id on the object; the generation annotation was left uncorrected'
)
return
}
if (liveOperationId !== input.intent.operationId) {
rootLogger.warn(
{
event: 'administrative_intent_generation_reassigned',
plural: input.plural,
namespace: input.namespace,
name: input.name,
writtenGeneration: persisted,
liveGeneration: snapshot.metadata?.generation,
},
'another administrative operation owns the intent annotation; this prediction was left uncorrected'
)
return
}
const resourceVersion = snapshot.metadata?.resourceVersion
if (!resourceVersion) {
governedTraceAdministrativeIntentReconcileFailedTotal.inc({
reason: 'no_precondition',
})
rootLogger.warn(
{
event: 'administrative_intent_generation_no_precondition',
plural: input.plural,
namespace: input.namespace,
name: input.name,
predictedGeneration: input.predictedGeneration,
persistedGeneration: persisted,
},
'no metadata.resourceVersion to use as a patch precondition; the intent annotation was left at the predicted value'
)
return
}
if (snapshot.metadata?.generation !== persisted) {
// A non-administrative writer bumped the spec without touching the
// annotations, so the object is now past the generation this operation
// produced. Correct it ANYWAY: `persisted` is the generation this write
// actually produced, which stays true wherever the object has got to,
// and writing it lands `annotated < live` — the terminal refusal the
// resolver exists to make.
//
// Leaving the PREDICTION in place is the dangerous option, and it is
// the one this branch used to take. `predicted` is by construction the
// next generation the object is most likely to reach, so a concurrent
// bump can make it match the live generation exactly, and then the
// resolver binds another writer's change to this operator's intent —
// the audit-trail falsification #329 exists to prevent, reintroduced by
// the code meant to prevent it.
rootLogger.warn(
{
event: 'administrative_intent_generation_outpaced',
plural: input.plural,
namespace: input.namespace,
name: input.name,
writtenGeneration: persisted,
liveGeneration: snapshot.metadata?.generation,
},
'object advanced past the generation this operation produced; correcting the annotation to it anyway'
)
}
try {
await this.customApi.patchNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: input.namespace,
plural: input.plural,
name: input.name,
body: {
metadata: {
resourceVersion,
annotations: {
[ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION]: String(persisted),
},
},
},
},
k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
)
return
} catch (err) {
if (extractK8sStatus(err) === 409 && attempt < maxAttempts) continue
governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'patch_failed' })
rootLogger.warn(
{
event: 'administrative_intent_generation_patch_failed',
plural: input.plural,
namespace: input.namespace,
name: input.name,
predictedGeneration: input.predictedGeneration,
persistedGeneration: persisted,
status: extractK8sStatus(err),
},
'could not correct the intent annotation to the persisted generation'
)
return
}
}
}
/**
* Merge-patch ONLY metadata.annotations of a custom object.
*
* Unlike updateResource/mutateResource this is a single server-side JSON
* merge patch: no read-modify-write, no resourceVersion, no retry loop —
* concurrent full replaces (e.g. the admin facade) cannot clobber it and it
* cannot clobber them. Used for write-only annotation projections such as
* `clerum.io/wake-requested`.
*/
async patchResourceAnnotations(
plural: ClerumResourceType,
name: string,
annotations: Record<string, string>,
namespace?: string
): Promise<unknown> {
const ns = this.resolveNamespace(plural, namespace)
try {
return await this.customApi.patchNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
body: { metadata: { annotations } },
},
k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
)
} catch (err) {
if (extractK8sStatus(err) === 404) {
throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
}
throw err
}
}
/**
* Monotonically project a single numeric annotation whose value must never
* regress under concurrent writers.
*
* Why this exists (and why a blind `patchResourceAnnotations` merge patch is
* unsafe here): the wake generation is a strictly monotonic counter in
* Postgres (the source of truth). The annotation is only a WRITE-ONLY
* PROJECTION of that counter for HCC to observe. A blind merge patch carries
* no ordering guarantee — with coalesce-window = 0 or control-api replicas
* > 1, a slower gen=5 patch can land AFTER a faster gen=6 patch and REGRESS
* the projected annotation. HCC advances `wakeHandledGeneration` up to the
* value it observes; a regressed projection can briefly exceed the projected
* value and SUPPRESS a later legitimate wake event. Bounded (the host still
* wakes via resync) but it violates the "strictly monotonic projection"
* invariant this projection is supposed to uphold.
*
* Fix: max-semantics with an optimistic-concurrency precondition.
* - Read the current projected value. If it is already >= the new
* generation, do NOT patch (never regress).
* - Otherwise merge-patch the annotation WITH the object's
* `metadata.resourceVersion` as a precondition. The apiserver rejects
* the patch with 409 if the object changed since the read; on 409 we
* re-read and re-evaluate max-semantics, which also lets a concurrent
* higher-generation write win the race.
* The generation itself always comes from Postgres — this method only
* decides WHETHER (and with which precondition) to project it.
*/
async patchAnnotationMonotonic(
plural: ClerumResourceType,
name: string,
annotationKey: string,
generation: number,
namespace?: string
): Promise<unknown> {
if (!Number.isFinite(generation) || generation < 0 || !Number.isInteger(generation)) {
throw new Error(
`patchAnnotationMonotonic: generation must be a non-negative integer, got ${generation}`
)
}
const ns = this.resolveNamespace(plural, namespace)
const maxAttempts = 5
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const current = (await this.getResource(plural, name, ns)) as {
metadata?: {
annotations?: Record<string, string>
resourceVersion?: string
}
}
const projected = parseProjectedGeneration(current.metadata?.annotations?.[annotationKey])
if (projected !== null && projected >= generation) {
// The annotation already projects an equal-or-higher generation.
// Patching would regress (lower value) or be a no-op (equal value):
// in both cases skip, preserving the monotonic invariant.
return current
}
const resourceVersion = current.metadata?.resourceVersion
if (!resourceVersion) {
// Fail loud: without a resourceVersion we cannot enforce the
// optimistic-concurrency precondition, so a blind patch could regress
// the projection — exactly the bug this method exists to prevent.
throw new Error(
`patchAnnotationMonotonic: ${plural}/${name} has no metadata.resourceVersion; cannot project ${annotationKey} safely`
)
}
try {
// metadata.resourceVersion in a merge-patch body is honored by the
// apiserver as an optimistic-concurrency precondition: the patch is
// rejected with 409 if the object changed since the read above.
return await this.customApi.patchNamespacedCustomObject(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
body: {
metadata: {
resourceVersion,
annotations: { [annotationKey]: String(generation) },
},
},
},
k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
)
} catch (err) {
const status = extractK8sStatus(err)
if (status === 404) {
throw new K8sNotFoundError(`${plural}/${name} not found in namespace ${ns}`)
}
if (status === 409 && attempt < maxAttempts) {
// The object changed under us (concurrent projection or unrelated
// metadata write). Re-read and re-evaluate max-semantics.
continue
}
throw err
}
}
throw new Error(
`patchAnnotationMonotonic: failed to project ${annotationKey}=${generation} onto ${plural}/${name} after ${maxAttempts} attempts`
)
}
async mutateResource(
plural: ClerumResourceType,
name: string,
mutate: (current: MutableResourceSnapshot) => ResourceMutation | Promise<ResourceMutation>,
namespace?: string
): Promise<unknown> {
const ns = this.resolveNamespace(plural, namespace)
let intent: HostAdministrativeIntent | null = null
const maxAttempts = 3
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
const current = (await this.getResource(plural, name, ns)) as MutableResourceSnapshot
const next = await mutate(current)
if (!next) {
return current
}
if (!intent) {
intent = await persistHostIntent({ plural, action: 'config', namespace: ns, name })
}
const sanitizedMetadata = stripAdministrativeIntentAnnotation(next.metadata)
// Same per-key merge as updateResource: platform-owned clerum.io/*
// annotations survive unless the mutation explicitly sets that key.
const mergedAnnotations = mergeAnnotationsForReplace(
current.metadata?.annotations,
sanitizedMetadata?.annotations
)
// Predicted, not persisted. This path is the likeliest producer of drift
// in the repository: it does not short-circuit on an unchanged spec (the
// `if (!next)` above only fires when the mutation returns nothing), so a
// mutation that computes the same spec still issues a replace, and the
// Host CRD's status subresource means such a replace leaves the
// generation untouched while the prediction moves ahead of it.
const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
const annotations = intent
? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
: mergedAnnotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
kind: kindFromPlural(plural),
metadata: {
name,
namespace: ns,
...(current.metadata?.labels && { labels: current.metadata.labels }),
...(sanitizedMetadata?.labels && { labels: sanitizedMetadata.labels }),
...(annotations && { annotations }),
...(current.metadata?.ownerReferences && {
ownerReferences: current.metadata.ownerReferences,
}),
...(current.metadata?.finalizers && { finalizers: current.metadata.finalizers }),
},
spec: next.spec,
}
let replaced: unknown
try {
replaced = await this.customApi.replaceNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
body: {
...resource,
metadata: {
...resource.metadata,
resourceVersion: current.metadata?.resourceVersion,
},
},
})
} catch (err) {
if (extractK8sStatus(err) === 409 && attempt < maxAttempts) {
continue
}
await persistHostFailure(intent)
throw err
}
await this.reconcileAdministrativeIntentGeneration({
plural,
name,
namespace: ns,
intent,
predictedGeneration,
written: replaced,
})
return replaced
}
throw new Error(`Failed to mutate ${plural}/${name} after ${maxAttempts} attempts`)
}
async deleteResource(
plural: ClerumResourceType,
name: string,
namespace?: string,
precondition?: ResourcePreconditions
): Promise<unknown> {
const ns = this.resolveNamespace(plural, namespace)
const intent = await persistHostIntent({ plural, action: 'delete', namespace: ns, name })
try {
const deleted = await this.customApi.deleteNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
...(precondition && (precondition.uid || precondition.resourceVersion)
? {
body: {
preconditions: {
...(precondition.uid ? { uid: precondition.uid } : {}),
...(precondition.resourceVersion
? { resourceVersion: precondition.resourceVersion }
: {}),
},
},
}
: {}),
})
if (intent && administrativeOperationService) {
await administrativeOperationService.persistHostOutcome(intent, 'succeeded', 'deleted')
}
return deleted
} catch (err) {
await persistHostFailure(intent)
throw err
}
}
// Merge-patches a CRD's /status subresource. Used by the recipe retry
// endpoint to drive the state-machine transition out of the terminal
// `failed` phase without going through a full spec update.
async patchResourceStatus(
plural: ClerumResourceType,
name: string,
statusPatch: Record<string, unknown>,
namespace?: string
): Promise<unknown> {
const ns = this.resolveNamespace(plural, namespace)
return this.customApi.patchNamespacedCustomObjectStatus(
{
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
plural,
name,
body: { status: statusPatch },
},
{ middleware: [k8s.setHeaderMiddleware('Content-Type', 'application/merge-patch+json')] }
)
}
}