Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(tracing): give the intent-drift tests a witness they cannot fake
Alfredo Lopez65ce0a5a5dad
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 6ae781fe2..7f6ce4328 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.595","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.595","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 5b35eb47d..860e1b21b 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.594",+ "version": "0.1.595","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 170ba8483..44b326f20 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -409,9 +409,16 @@ export const governedTraceConflictingTotal = getOrCreateCounter({/*** A Host whose administrative-intent annotation names a generation the object* has already passed, so no outcome for it can ever be attributed (#329). The- * authoritative drift signal: HCC's own reporter counter labels this- * `result="rejected"`, the same label it uses for malformed input, and cannot- * tell the two apart.+ * authoritative signal for drift being OBSERVED: HCC's own reporter counter+ * labels this `result="rejected"`, the same label it uses for malformed input,+ * and cannot tell the two apart.+ *+ * Read it as an arrival rate, not as a population. Once HCC classifies the+ * refusal as terminal it stops resubmitting, so a Host that stays drifted stops+ * incrementing this counter — a standing drift is invisible here by design, and+ * `increase(...[1h])` going to zero means the loop was cut, not that the Host+ * was repaired. Nothing in this service currently gauges how many Hosts are+ * drifted right now.** `namespace` is the only label. Host names are unbounded, so putting one in a* label would make the series count grow with the cluster.@@ -422,6 +429,28 @@ export const governedTraceAdministrativeIntentDriftTotal = getOrCreateCounter({labelNames: ['namespace'] as const as Array<'namespace'>,})+/**+ * The write path failed to correct a predicted generation it knows is wrong+ * (#329 layer 3). Every increment means an administrative intent annotation was+ * left pinned to a value the object never reached, which the resolver will+ * later refuse — so this counter leads the drift counter above.+ *+ * It exists because the repair failures are otherwise invisible: the reconcile+ * is documented never to throw, so each one is a single `warn` line with no+ * metric. A revoked `patch` verb or an admission webhook that rejects+ * annotation writes would degrade the administrative audit trail silently,+ * with the drift counter naming the symptom and nothing naming the cause.+ *+ * `reason` is a closed set of four fixed strings, matching the `event` field of+ * the log line emitted beside it. No Host identity: names are unbounded, and+ * the namespace is already carried by the drift counter.+ */+export const governedTraceAdministrativeIntentReconcileFailedTotal = getOrCreateCounter({+ name: 'governed_trace_administrative_intent_reconcile_failed_total',+ help: 'Count of administrative intent generation corrections that could not be applied, labelled by reason.',+ labelNames: ['reason'] as const as Array<'reason'>,+})+export const governedTraceIngestDurationSeconds = getOrCreateHistogram({name: 'governed_trace_ingest_duration_seconds',help: 'Duration of governed trace ingestion and stream registration.',diff --git a/control-api/src/services/resourceService.ts b/control-api/src/services/resourceService.tsindex ca25ca2cb..c0f4f45d9 100644--- a/control-api/src/services/resourceService.ts+++ b/control-api/src/services/resourceService.ts@@ -1,5 +1,6 @@import * as k8s from '@kubernetes/client-node'import { rootLogger } from '../observability/logger.js'+import { governedTraceAdministrativeIntentReconcileFailedTotal } from '../observability/metrics.js'import {CLERUM_GROUP,CLERUM_VERSION,@@ -13,7 +14,10 @@ import {kindFromPlural,parseProjectedGeneration,} from './resourceServiceHelpers.js'-import { ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION } from './tracing/adminOperationConstants.js'+import {+ ADMINISTRATIVE_INTENT_ANNOTATION,+ ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION,+} from './tracing/adminOperationConstants.js'import { currentAdministrativeRequestContext } from './tracing/adminOperationContext.js'import {ControlApiAdministrativeOperationService,@@ -255,9 +259,12 @@ export class ResourceService {name: body.metadata.name,})const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata) ?? body.metadata- // A create is the one prediction that is almost always right — the- // apiserver assigns generation 1 — but it is still a prediction, and an- // admission webhook that mutates the spec can land the object at 2.+ // A create is the one prediction that cannot be wrong: the apiserver's+ // `PrepareForCreate` sets generation 1 unconditionally, and it runs AFTER+ // mutating admission, so no webhook can land the object anywhere else. The+ // value still goes through `reconcileAdministrativeIntentGeneration` below,+ // which returns without patching once it confirms 1 === 1 — the check is+ // what makes the invariant observable instead of assumed.const predictedGeneration = 1const annotations = intent? withAdministrativeIntentAnnotation(@@ -440,16 +447,27 @@ export class ResourceService {* Replace the PREDICTED generation in the administrative-intent annotation* with the one the apiserver actually persisted (#329).*- * The three write paths above annotate with `current.generation + 1` before- * the write, because the annotation has to travel inside the same body. That- * prediction is wrong whenever the write does not bump the generation, and- * the Host CRD declares a status subresource- * (`charts/clerum-crds/crds/host.yaml`), so `metadata.generation` advances- * ONLY on spec changes. A replace whose spec is byte-identical — an- * idempotent operator PUT, a `mutateResource` whose mutation is a no-op, a- * hook-ref resync — leaves the object at N while the annotation says N+1.- * Nothing ever retires that annotation, so the Host stops binding outcomes- * forever.+ * The three callers — `createResource` (:261), `updateResource` (:375) and+ * `mutateResource` (:766) — annotate BEFORE the write, because the annotation+ * has to travel inside the same body. `createResource` predicts the literal+ * `1`; the other two predict `current.generation + 1`. That prediction is+ * wrong whenever the write does not bump the generation, and the Host CRD+ * declares a status subresource (`charts/clerum-crds/crds/host.yaml`), so+ * `metadata.generation` advances ONLY on spec changes. A replace whose spec+ * is byte-identical — an idempotent operator PUT, a `mutateResource` whose+ * mutation is a no-op, a hook-ref resync — leaves the object at N while the+ * annotation says N+1. Nothing ever retires that annotation, so the Host+ * stops binding outcomes forever.+ *+ * The correction only ever LOWERS the annotation, and that bound is what+ * lets the binding resolver treat `annotated < live` as terminal. Every+ * replace carries a `resourceVersion` precondition (:404, :795), so the+ * apiserver's `old` object is the one this service read and+ * `PrepareForUpdate` can raise the generation by at most one: `persisted` is+ * therefore never greater than `predicted`. On a create it is neither —+ * `PrepareForCreate` sets generation 1 unconditionally, after mutating+ * admission has already run, so `persisted === predicted === 1` and the+ * guard below returns before any patch is issued.** A merge-patch of annotations alone does not bump the generation, so this* correction cannot feed itself.@@ -474,9 +492,12 @@ export class ResourceService {const metadata = (input.written as { metadata?: { generation?: number } } | null)?.metadataconst persisted = metadata?.generationif (typeof persisted !== 'number' || !Number.isSafeInteger(persisted)) {+ governedTraceAdministrativeIntentReconcileFailedTotal.inc({+ reason: 'generation_missing',+ })rootLogger.warn({- event: 'administrative_intent_generation_unverified',+ event: 'administrative_intent_generation_missing',plural: input.plural,namespace: input.namespace,name: input.name,@@ -492,20 +513,25 @@ export class ResourceService {for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {// The first attempt uses the write's own response; a retry re-reads,// because a 409 means the resourceVersion we held is stale.- let snapshot: { metadata?: { generation?: number; resourceVersion?: string } }+ let snapshot: MutableResourceSnapshotif (attempt === 1) {- snapshot = input.written as { metadata?: { generation?: number; resourceVersion?: string } }+ snapshot = input.written as MutableResourceSnapshot} else {try {- snapshot = (await this.getResource(input.plural, input.name, input.namespace)) as {- metadata?: { generation?: number; resourceVersion?: string }- }+ snapshot = (await this.getResource(+ input.plural,+ input.name,+ input.namespace+ )) as MutableResourceSnapshot} catch (err) {- // The only place in this method that can throw. Caught so the- // documented contract holds, and reported rather than swallowed.+ // One of the two places in this method that can throw — the patch+ // below is the other, which is why it carries its own catch. Both are+ // caught so the documented contract holds, and both report the reason+ // rather than swallowing it.+ governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'reread_failed' })rootLogger.warn({- event: 'administrative_intent_generation_patch_failed',+ event: 'administrative_intent_generation_reread_failed',plural: input.plural,namespace: input.namespace,name: input.name,@@ -518,30 +544,54 @@ export class ResourceService {return}}- if (snapshot.metadata?.generation !== persisted) {- // Something else advanced the object past the generation this- // operation produced. Writing `persisted` now would attribute a- // generation this intent did not cause; writing the newer one would- // do the same. The intent is genuinely unattributable from here, and- // the binding resolver refuses its outcomes by design (#329).+ // Ownership, not generation, decides whether this correction is still+ // ours to make. The two annotation keys are written as a PAIR+ // (`withAdministrativeIntentAnnotation`), so once another administrative+ // operation has landed, the id key names ITS operation and patching the+ // generation key alone would fuse its id to our generation — a pair that+ // never existed. Its own reconcile will correct its own prediction.+ const liveOperationId = snapshot.metadata?.annotations?.[ADMINISTRATIVE_INTENT_ANNOTATION]+ if (liveOperationId === undefined) {+ // Absent is not reassigned, and the two must not share a log line. No+ // id key means no obligation is pending on this object at all, so+ // writing the generation key alone would create half a pair that names+ // no operation — worse than the prediction it replaces.+ governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'intent_absent' })+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_intent_absent',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ predictedGeneration: input.predictedGeneration,+ persistedGeneration: persisted,+ },+ 'no administrative intent id on the object; the generation annotation was left uncorrected'+ )+ return+ }+ if (liveOperationId !== input.intent.operationId) {rootLogger.warn({- event: 'administrative_intent_generation_superseded',+ event: 'administrative_intent_generation_reassigned',plural: input.plural,namespace: input.namespace,name: input.name,writtenGeneration: persisted,liveGeneration: snapshot.metadata?.generation,},- 'object advanced past the generation this operation produced; intent annotation left uncorrected'+ 'another administrative operation owns the intent annotation; this prediction was left uncorrected')return}const resourceVersion = snapshot.metadata?.resourceVersionif (!resourceVersion) {+ governedTraceAdministrativeIntentReconcileFailedTotal.inc({+ reason: 'no_precondition',+ })rootLogger.warn({- event: 'administrative_intent_generation_unverified',+ event: 'administrative_intent_generation_no_precondition',plural: input.plural,namespace: input.namespace,name: input.name,@@ -552,6 +602,33 @@ export class ResourceService {)return}+ if (snapshot.metadata?.generation !== persisted) {+ // A non-administrative writer bumped the spec without touching the+ // annotations, so the object is now past the generation this operation+ // produced. Correct it ANYWAY: `persisted` is the generation this write+ // actually produced, which stays true wherever the object has got to,+ // and writing it lands `annotated < live` — the terminal refusal the+ // resolver exists to make.+ //+ // Leaving the PREDICTION in place is the dangerous option, and it is+ // the one this branch used to take. `predicted` is by construction the+ // next generation the object is most likely to reach, so a concurrent+ // bump can make it match the live generation exactly, and then the+ // resolver binds another writer's change to this operator's intent —+ // the audit-trail falsification #329 exists to prevent, reintroduced by+ // the code meant to prevent it.+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_outpaced',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ writtenGeneration: persisted,+ liveGeneration: snapshot.metadata?.generation,+ },+ 'object advanced past the generation this operation produced; correcting the annotation to it anyway'+ )+ }try {await this.customApi.patchNamespacedCustomObject({@@ -574,6 +651,7 @@ export class ResourceService {return} catch (err) {if (extractK8sStatus(err) === 409 && attempt < maxAttempts) continue+ governedTraceAdministrativeIntentReconcileFailedTotal.inc({ reason: 'patch_failed' })rootLogger.warn({event: 'administrative_intent_generation_patch_failed',diff --git a/control-api/src/services/resourceServiceHelpers.ts b/control-api/src/services/resourceServiceHelpers.tsindex b529dc3e7..9f2bcfea3 100644--- a/control-api/src/services/resourceServiceHelpers.ts+++ b/control-api/src/services/resourceServiceHelpers.ts@@ -1,11 +1,32 @@import type { ClerumResourceType } from '../types.js'+/**+ * HTTP status of a Kubernetes-ish error, or null when none can be read.+ *+ * `.httpStatus` is checked LAST, and the position is the point: every earlier+ * field wins as before, so no call site that reads a number today reads a+ * different one after. The only behaviour that changes is where this returned+ * `null` for a status it could have known.+ *+ * That gap was real. `httpStatus` is what this service's OWN error classes+ * carry (`K8sNotFoundError`, `K8sConflictError` in resourceService.ts), and+ * they set nothing else — so `getResource` rejecting with a wrapped 404 read as+ * "unknown status" here. `routes/admin/resources.ts:788` works around exactly+ * that with a hand-written `instanceof K8sNotFoundError` check; the repo's three+ * other status extractors (`http/k8sError.ts:18`, `http/errorHandler.ts:54`,+ * `k8s.ts:719`) all read the field already. This one was the outlier.+ *+ * Existing 404/409 call sites are unaffected: they catch errors thrown by the+ * raw `@kubernetes/client-node` client, whose `ApiException` sets `.code`, and+ * a `K8sNotFoundError` cannot reach them.+ */export function extractK8sStatus(err: unknown): number | null {if (!err || typeof err !== 'object') return nullconst maybe = err as {statusCode?: numbercode?: numberresponse?: { statusCode?: number; status?: number }+ httpStatus?: number}if (typeof maybe.statusCode === 'number') return maybe.statusCodeif (typeof maybe.code === 'number') return maybe.code@@ -13,6 +34,7 @@ export function extractK8sStatus(err: unknown): number | null {return maybe.response.statusCode}if (maybe.response && typeof maybe.response.status === 'number') return maybe.response.status+ if (typeof maybe.httpStatus === 'number') return maybe.httpStatusreturn null}diff --git a/control-api/src/services/tracing/adminOperationBindingResolver.ts b/control-api/src/services/tracing/adminOperationBindingResolver.tsindex 9bd75c6ec..e173795b0 100644--- a/control-api/src/services/tracing/adminOperationBindingResolver.ts+++ b/control-api/src/services/tracing/adminOperationBindingResolver.ts@@ -51,6 +51,27 @@ const STATUS_REF =/^host:([a-z0-9]([-a-z0-9]*[a-z0-9])?)\/([a-z0-9]([-a-z0-9]*[a-z0-9])?):generation=([1-9][0-9]*):uid=([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i+/**+ * A generation annotation is a decimal integer >= 1, spelled the way the+ * apiserver spells `metadata.generation` — the same shape `STATUS_REF` above+ * requires of the wire value it is compared against.+ *+ * `Number()` is not usable here. `Number('')`, `Number(' ')` and `Number('\n')`+ * are all `0`, which passes `Number.isSafeInteger` and is strictly below every+ * live generation, so an empty or whitespace-only annotation would be+ * classified as DRIFT and refused TERMINALLY: an operator action named as+ * superseded when the annotation is merely malformed, counted by the very+ * metric that exists to tell those two apart. `'0'` and any negative spelling+ * are rejected for the same reason — generations start at 1, so neither can be+ * a generation control-api ever wrote. `'0x3'`, `' 3 '` and `'3e0'` are+ * rejected because a value this key's sole writer never produces is evidence of+ * tampering, not of drift.+ *+ * Unparseable therefore means "no usable annotation", which is the retryable+ * `null`, never a terminal refusal.+ */+const GENERATION_ANNOTATION = /^[1-9][0-9]*$/+const DRIFT_REFUSAL: BindingRefusal = { refusal: 'administrative_intent_generation_drift' }type Candidate = {@@ -127,18 +148,29 @@ export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOp)return nullconst operationId = metadata.annotations?.[ADMINISTRATIVE_INTENT_ANNOTATION]- const expectedGeneration = Number(+ const expectedGeneration = parseGenerationAnnotation(metadata.annotations?.[ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION])const targetRef = `${reference.namespace}/${reference.name}`- if (!operationId || !UUID.test(operationId) || !Number.isSafeInteger(expectedGeneration))- return null+ if (!operationId || !UUID.test(operationId) || expectedGeneration === null) return null+ // Classify the event by its own defect BEFORE classifying the object's.+ // An event whose payload cannot be read is malformed input, not evidence+ // that the annotation drifted, and answering it with the drift code would+ // both mis-name it and count it in the drift metric. This keeps the+ // pre-#329 answer (retryable) for a payload the caller built wrong.+ const outcome = safeOutcome(events[index]!.payload?.status)+ if (!outcome) return nullif (expectedGeneration < reference.generation) {// Every identity field above already matched, so the live object IS the// one the reporter observed, and its generation has moved past the one- // the annotation pins. Generations only grow and nothing ever retires- // the annotation, so this pair can never bind again: terminal, not- // "unavailable right now".+ // the annotation pins. The pair can never bind again: the live+ // generation only grows, and NOTHING EVER RAISES THE ANNOTATION to+ // catch up with it. `reconcileAdministrativeIntentGeneration` only ever+ // lowers a prediction to the generation the write actually produced —+ // every replace carries a `resourceVersion` precondition+ // (`resourceService.ts:404`, `:795`), so the persisted generation can+ // never exceed the predicted one — and no other writer owns this key.+ // Terminal, not "unavailable right now".this.metrics.inc({ namespace: reference.namespace })// The Host name goes to the log, never to the metric label: names are// unbounded and a label carrying one grows the series count with the@@ -159,16 +191,16 @@ export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOp// window between control-api predicting a generation and the corrective// annotation patch landing, and it resolves on its own.if (expectedGeneration !== reference.generation) return null- const outcome = safeOutcome(events[index]!.payload?.status)- if (!outcome) return nullreturn { reference, operationId, targetRef, outcome }})const intentInputs = candidates- .filter(candidate => candidate !== null && !isBindingRefusal(candidate))+ .filter(+ (candidate): candidate is Candidate => candidate !== null && !isBindingRefusal(candidate)+ ).map(candidate => ({- operationId: (candidate as Candidate).operationId,- targetRef: (candidate as Candidate).targetRef,- namespace: (candidate as Candidate).reference.namespace,+ operationId: candidate.operationId,+ targetRef: candidate.targetRef,+ namespace: candidate.reference.namespace,}))const intents = await this.intentLookup.findHostIntents(intentInputs)return candidates.map(candidate => {@@ -224,6 +256,13 @@ function parseHostStatusRef(value: string | undefined): {}}+/** See `GENERATION_ANNOTATION` for why `Number()` alone is not usable here. */+function parseGenerationAnnotation(raw: string | undefined): number | null {+ if (raw === undefined || !GENERATION_ANNOTATION.test(raw)) return null+ const parsed = Number(raw)+ return Number.isSafeInteger(parsed) ? parsed : null+}+function safeOutcome(value: unknown): 'succeeded' | 'failed' | null {return value === 'succeeded' || value === 'failed' ? value : null}diff --git a/control-api/src/services/tracing/routeSubmissionService.ts b/control-api/src/services/tracing/routeSubmissionService.tsindex f3ecab482..0fee4dacf 100644--- a/control-api/src/services/tracing/routeSubmissionService.ts+++ b/control-api/src/services/tracing/routeSubmissionService.ts@@ -106,15 +106,26 @@ export interface InfrastructureWorkloadBindingResolver {): MaybePromise<readonly (InfrastructureTelemetryServerBindingV1 | null | undefined)[]>}+/**+ * `BindingRefusal` is deliberately NOT part of this contract. A refusal is+ * administrative-specific — `resolveTrustedBindings` answers one with+ * `AdministrativeIntentGenerationDriftError`, whose code and message name a+ * Host annotation — so a `run` or `workload` resolver that returned one would+ * produce a 409 describing a subsystem its caller has nothing to do with, and+ * HCC's `TERMINAL_RESPONSES` would classify it terminal for the wrong family.+ *+ * Refusals reach `resolveTrustedBindings` only through a resolver that puts+ * `BindingRefusal` in its own `Binding` type (today: exactly+ * `AdministrativeOperationBindingResolver`), and the `Exclude` on the return+ * type strips them back out. The compiler, not a reviewer, is what keeps the+ * other two families out.+ */type BindingResolver<Principal, Event, Binding> = {- resolve(- principal: Principal,- event: Event- ): MaybePromise<Binding | BindingRefusal | null | undefined>+ resolve(principal: Principal, event: Event): MaybePromise<Binding | null | undefined>resolveMany?(principal: Principal,events: readonly Event[]- ): MaybePromise<readonly (Binding | BindingRefusal | null | undefined)[]>+ ): MaybePromise<readonly (Binding | null | undefined)[]>}export class TracingBindingUnavailableError extends Error {@@ -133,10 +144,14 @@ export class TracingBindingUnavailableError extends Error {/*** The reported object carries an administrative-intent annotation pinned to a- * generation the object has already passed, so no future report can ever bind:- * the reporter always reads the live object and `metadata.generation` only- * grows. Deterministic by construction, which is why it is a 409 with a- * forwarded code and not the retryable 403 that `TracingBindingUnavailableError`+ * generation the object has already passed, so no future report can ever bind.+ * Two facts make that permanent, and the second is the load-bearing one: the+ * reporter always reads the live object and `metadata.generation` only grows,+ * AND nothing ever raises the annotation to catch up with it — control-api's+ * corrective patch only ever lowers a prediction (see+ * `adminOperationBindingResolver.ts`), and no other writer owns the key.+ * Deterministic by construction, which is why it is a 409 with a forwarded+ * code and not the retryable 403 that `TracingBindingUnavailableError`* produces.** The message carries no Host identity: `name`/`namespace`/`uid` go to the@@ -415,7 +430,7 @@ async function resolveTrustedBindings<Principal, Event, Binding>(input: {principal: Principalevents: readonly Event[]bindingKind: 'run' | 'operation' | 'workload'-}): Promise<Binding[]> {+}): Promise<Exclude<Binding, BindingRefusal>[]> {const resolved = input.resolver.resolveMany? await input.resolver.resolveMany(input.principal, input.events): await Promise.all(input.events.map(event => input.resolver.resolve(input.principal, event)))@@ -428,10 +443,22 @@ async function resolveTrustedBindings<Principal, Event, Binding>(input: {// before any append runs, so a batch holding one drifted event is refused// whole and nothing is stored. A refusal only picks a different status than// `null` does; it does not let the batch continue.+ //+ // The refusal is scanned for across the WHOLE batch before the index-ordered+ // walk below, because the two answers are not interchangeable: a refusal is+ // terminal and a `null` is retryable. Letting the walk decide would let a+ // `null` at a lower index mask a refusal at a higher one and answer a batch+ // that can never succeed with the code-less, retryable 403 — re-creating the+ // permanent re-enqueue loop #329 exists to end, for the whole batch. HCC+ // posts one event per request today+ // (`administrativeOutcomeReporter.ts:144-161`), so no live caller can build+ // such a batch; pinning the order here means that stops being the reason+ // this is correct.+ const refusalIndex = resolved.findIndex(binding => isBindingRefusal(binding))+ if (refusalIndex >= 0) throw new AdministrativeIntentGenerationDriftError(refusalIndex)return resolved.map((binding, index) => {- if (isBindingRefusal(binding)) throw new AdministrativeIntentGenerationDriftError(index)if (!binding) throw new TracingBindingUnavailableError(input.bindingKind, index)- return binding+ return binding as Exclude<Binding, BindingRefusal>})}diff --git a/control-api/test/observability.tracingMetrics.test.ts b/control-api/test/observability.tracingMetrics.test.tsindex 88bc3b610..1af783f96 100644--- a/control-api/test/observability.tracingMetrics.test.ts+++ b/control-api/test/observability.tracingMetrics.test.ts@@ -69,6 +69,18 @@ const tracingMetrics = [name: 'governed_trace_last_error_timestamp_seconds',labelNames: ['scope', 'reason'],},+ // #329. `namespace` is the only label either carries, and the+ // high-cardinality assertion below is why: the drift log names the Host, the+ // metric never does, because a label carrying a resource name grows the+ // series count with the cluster.+ {+ name: 'governed_trace_administrative_intent_drift_total',+ labelNames: ['namespace'],+ },+ {+ name: 'governed_trace_administrative_intent_reconcile_failed_total',+ labelNames: ['reason'],+ },] as constdescribe('governed tracing metric registration', () => {diff --git a/control-api/test/routes.tracingSubmissionBoundary.test.ts b/control-api/test/routes.tracingSubmissionBoundary.test.tsindex 7f37bc2f0..2d58f5dc0 100644--- a/control-api/test/routes.tracingSubmissionBoundary.test.ts+++ b/control-api/test/routes.tracingSubmissionBoundary.test.ts@@ -612,12 +612,17 @@ describe('internal tracing submission routers — rejection code on the wire', (*/describe('administrative intent generation drift (#329)', () => {const LIVE_REF = `host:mcp-host/chatllm:generation=7:uid=${HOST_UID}`- const DRIFTED_REF = `host:mcp-host/drifted:generation=7:uid=${HOST_UID}`+ // The Host name shares no substring with any code or message the response+ // can legitimately carry. Naming it `drifted` made the leak assertion below+ // one letter from `administrative_intent_generation_drift`, so a future+ // message wording "the annotation drifted past" would fail this test for a+ // reason that has nothing to do with identity leaking.+ const DRIFTED_REF = `host:mcp-host/canary-host:generation=7:uid=${HOST_UID}`it('answers a superseded intent annotation 409 with its code through the real route', async () => {const { app, appendManyInTransaction } = realAdministrativeApp([hostResource('chatllm', 7, 7),- hostResource('drifted', 7, 6),+ hostResource('canary-host', 7, 6),])// Liveness: the healthy Host binds and stores through this same app, so@@ -640,7 +645,7 @@ describe('internal tracing submission routers — rejection code on the wire', (})it('keeps Host identity out of the refusal body', async () => {- const { app } = realAdministrativeApp([hostResource('drifted', 7, 6)])+ const { app } = realAdministrativeApp([hostResource('canary-host', 7, 6)])const refused = await postOutcomes(app, [DRIFTED_REF])@@ -649,7 +654,7 @@ describe('internal tracing submission routers — rejection code on the wire', (// the name below is not an unrelated failure with an empty body.expect(refused.body.code).toBe('administrative_intent_generation_drift')const serialized = JSON.stringify(refused.body)- expect(serialized).not.toContain('drifted')+ expect(serialized).not.toContain('canary-host')expect(serialized).not.toContain('mcp-host')expect(serialized).not.toContain(HOST_UID)})@@ -663,7 +668,7 @@ describe('internal tracing submission routers — rejection code on the wire', (it('refuses a batch whole when one of its events has drifted', async () => {const { app, listResource, appendManyInTransaction } = realAdministrativeApp([hostResource('chatllm', 7, 7),- hostResource('drifted', 7, 6),+ hostResource('canary-host', 7, 6),])const mixed = await postOutcomes(app, [LIVE_REF, DRIFTED_REF])@@ -676,5 +681,52 @@ describe('internal tracing submission routers — rejection code on the wire', (expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')expect(appendManyInTransaction).not.toHaveBeenCalled()})++ /**+ * Order must not decide the status. A batch can hold both an event that is+ * merely unresolvable (no such Host yet — retryable, 403) and one that has+ * drifted (terminal, 409), and scanning positionally answers with whichever+ * came first.+ *+ * Answering 403 there is the expensive mistake: HCC classifies a bare 403+ * as retryable, so it re-enqueues the whole batch, the drifted event comes+ * back with it, and the pair loops for as long as the annotation stands —+ * which is forever, because nothing retires it. The terminal refusal has to+ * win regardless of where it sits, so it is scanned for first.+ */+ it('answers a batch with the terminal refusal even when an unresolvable event precedes it', async () => {+ const { app, listResource, appendManyInTransaction } = realAdministrativeApp([+ hostResource('canary-host', 7, 6),+ ])+ const UNRESOLVABLE_REF = `host:mcp-host/not-created-yet:generation=7:uid=${HOST_UID}`++ const mixed = await postOutcomes(app, [UNRESOLVABLE_REF, DRIFTED_REF])++ // 409 with the code, not the 403 the leading unresolvable event would+ // have produced on its own.+ expect(mixed.status).toBe(409)+ expect(mixed.body.code).toBe('administrative_intent_generation_drift')+ // Liveness for the negative below: the resolver ran over the whole batch.+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')+ expect(appendManyInTransaction).not.toHaveBeenCalled()+ })++ /**+ * The companion row: with no refusal in the batch, an unresolvable event+ * still produces the retryable 403. Without it, a service that answered 409+ * for every unresolved batch would pass the test above.+ */+ it('still answers an unresolvable event with a retryable 403 when nothing drifted', async () => {+ const { app, appendManyInTransaction } = realAdministrativeApp([+ hostResource('chatllm', 7, 7),+ ])+ const UNRESOLVABLE_REF = `host:mcp-host/not-created-yet:generation=7:uid=${HOST_UID}`++ const refused = await postOutcomes(app, [UNRESOLVABLE_REF, LIVE_REF])++ expect(refused.status).toBe(403)+ expect(refused.body.code).toBeUndefined()+ expect(appendManyInTransaction).not.toHaveBeenCalled()+ })})})diff --git a/control-api/test/services.adminOperationBindingResolver.test.ts b/control-api/test/services.adminOperationBindingResolver.test.tsindex 5c68563ac..8cec1e12d 100644--- a/control-api/test/services.adminOperationBindingResolver.test.ts+++ b/control-api/test/services.adminOperationBindingResolver.test.ts@@ -307,19 +307,150 @@ describe('HccAdministrativeOutcomeBindingResolver', () => {it('does not report drift when the uid does not match', async () => {const metrics = { inc: vi.fn() }const other = host(undefined, 7, 6, '00000000-0000-4000-8000-000000000000')- const listResource = vi.fn().mockResolvedValue([other])+ other.metadata.name = 'chatllm-recreated'+ // The witness rides in the same call: a Host with a MATCHING uid whose+ // annotation is equally behind. `listResource` alone is not enough —+ // deleting the drift branch entirely leaves that spy just as satisfied,+ // and so does a STATUS_REF that never parsed. Only a refusal produced+ // beside the null proves the branch was armed in this arrangement.+ const drifted = host(undefined, 7, 6)+ const listResource = vi.fn().mockResolvedValue([other, drifted])++ const [recreated, superseded] = await new HccAdministrativeOutcomeBindingResolver(+ { getResource: vi.fn(), listResource },+ {+ findHostIntent: vi.fn(),+ findHostIntents: vi+ .fn()+ .mockResolvedValue(+ new Map([...intentMap(), ...intentMap(undefined, 'chatllm-recreated')])+ ),+ },+ { warn: vi.fn() } as never,+ metrics+ ).resolveMany(principal, [+ {+ ...event,+ sourceEventId: 'outcome-recreated',+ sourceStatusRef: `host:mcp-host/chatllm-recreated:generation=7:uid=${HOST_UID}`,+ },+ event,+ ])++ expect(superseded).toEqual({ refusal: 'administrative_intent_generation_drift' })+ expect(recreated).toBeNull()+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')+ // One refusal counted, not two: identity is checked before drift, so a+ // #694 recreation is never named as a superseded operator action.+ expect(metrics.inc).toHaveBeenCalledTimes(1)+ })++ /**+ * A malformed annotation is malformed input, not drift, and the difference+ * is terminal-vs-retryable.+ *+ * `Number()` alone cannot make it: `Number('')`, `Number(' ')` and+ * `Number('\n')` are all `0`, which passes `Number.isSafeInteger` and is+ * strictly below every live generation. Each of those rows would therefore+ * be read as `annotated < live`, refused TERMINALLY, and counted by the+ * very metric that exists to separate the two — an operator action named as+ * superseded on the strength of an empty string.+ *+ * The liveness witness is structural rather than a spy: the genuinely+ * drifted Host rides in the SAME `resolveMany` call as the malformed rows.+ * Its refusal proves the drift branch executed during this call, so every+ * `null` beside it is a decision the resolver reached, not a path it never+ * entered. A resolver that skipped the candidate loop would lose the+ * refusal too and fail on the first assertion.+ */+ it('answers a malformed generation annotation retryably, never as drift', async () => {+ const malformed = ['', ' ', '\n', '0', '-1', '0x3', ' 3 ', '3e0', '+3', '007']+ const metrics = { inc: vi.fn() }++ const hosts = malformed.map((raw, index) => {+ const entry = host(undefined, 7, 7)+ entry.metadata.name = `chatllm-${index}`+ entry.metadata.annotations['clerum.io/administrative-intent-generation'] = raw+ return entry+ })+ const drifted = host(undefined, 7, 6)++ const resolver = new HccAdministrativeOutcomeBindingResolver(+ {+ getResource: vi.fn(),+ listResource: vi.fn().mockResolvedValue([...hosts, drifted]),+ },+ {+ findHostIntent: vi.fn(),+ findHostIntents: vi+ .fn()+ .mockResolvedValue(+ new Map([+ ...intentMap(),+ ...malformed.flatMap((_, index) => [...intentMap(undefined, `chatllm-${index}`)]),+ ])+ ),+ },+ { warn: vi.fn() } as never,+ metrics+ )++ const results = await resolver.resolveMany(principal, [+ ...malformed.map((_, index) => ({+ ...event,+ sourceEventId: `outcome-${index}`,+ sourceStatusRef: `host:mcp-host/chatllm-${index}:generation=7:uid=${HOST_UID}`,+ })),+ { ...event, sourceEventId: 'outcome-drifted' },+ ])++ // The witness first: the drift path ran in this call.+ expect(results[malformed.length]).toEqual({+ refusal: 'administrative_intent_generation_drift',+ })+ expect(results.slice(0, malformed.length)).toEqual(malformed.map(() => null))+ // Exactly one refusal was counted — the real one. Every malformed row+ // stayed out of the metric, which is what makes the metric readable.+ expect(metrics.inc).toHaveBeenCalledTimes(1)+ })++ /**+ * The companion to the row above: a well-formed annotation equal to the+ * live generation still binds. Without this, tightening the parser until it+ * rejected everything would leave the suite green.+ */+ it('still binds a well-formed annotation that equals the live generation', async () => {+ const result = await new HccAdministrativeOutcomeBindingResolver(+ { getResource: vi.fn(), listResource: vi.fn().mockResolvedValue([host(undefined, 7, 7)]) },+ { findHostIntent: vi.fn(), findHostIntents: vi.fn().mockResolvedValue(intentMap()) },+ { warn: vi.fn() } as never,+ { inc: vi.fn() }+ ).resolve(principal, event)++ expect(result).toMatchObject({ action: 'host_mutation', operationId: OPERATION_ID })+ })++ /**+ * Ordering, not just classification. An event whose own payload cannot be+ * read is malformed INPUT; classifying the object's annotation first would+ * answer it with the drift code and count it, mis-naming a request the+ * caller built wrong as an operator action that was superseded.+ *+ * The Host here is genuinely drifted, so a resolver that classified the+ * object before the event would return the refusal and fail.+ */+ it('classifies an unreadable payload before it classifies the annotation', async () => {+ const metrics = { inc: vi.fn() }+ const listResource = vi.fn().mockResolvedValue([host(undefined, 7, 6)])const result = await new HccAdministrativeOutcomeBindingResolver({ getResource: vi.fn(), listResource },{ findHostIntent: vi.fn(), findHostIntents: vi.fn().mockResolvedValue(intentMap()) },{ warn: vi.fn() } as never,metrics- ).resolve(principal, event)+ ).resolve(principal, { ...event, payload: { resource_class: 'Host', status: 'pending' } })expect(result).toBeNull()- // Liveness witness for the negative assertion below: the resolver did- // run the lookup and reach the candidate loop, so the uncounted metric- // is a decision, not an unexecuted path.expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')expect(metrics.inc).not.toHaveBeenCalled()})diff --git a/control-api/test/services.resourceService.test.ts b/control-api/test/services.resourceService.test.tsindex 5409111fb..f7a76331a 100644--- a/control-api/test/services.resourceService.test.ts+++ b/control-api/test/services.resourceService.test.ts@@ -1,4 +1,5 @@import { describe, expect, it, vi } from 'vitest'+import { governedTraceAdministrativeIntentReconcileFailedTotal } from '../src/observability/metrics.js'import {K8sConflictError,K8sNotFoundError,@@ -670,7 +671,9 @@ describe('ResourceService Host administrative intent', () => {*/describe('ResourceService administrative intent generation reconciliation (#329)', () => {const GENERATION = 'clerum.io/administrative-intent-generation'+ const INTENT_ID = 'clerum.io/administrative-intent-id'const OPERATION_ID = '11111111-1111-4111-8111-111111111111'+ const OTHER_OPERATION_ID = '33333333-3333-4333-8333-333333333333'function withIntent() {setAdministrativeOperationService({@@ -687,47 +690,101 @@ describe('ResourceService administrative intent generation reconciliation (#329)}/**- * `replaced` is what the apiserver answers the replace with — the object as- * persisted. `generation: 4` against a live 4 is the unchanged-spec case.+ * What the apiserver answers a replace with: the object AS PERSISTED, which+ * therefore echoes back the annotation pair the replace just wrote.+ *+ * Carrying the annotations is not decoration. The correction checks ownership+ * on this object before patching, so a fixture that omitted them would drive+ * a branch the real apiserver can never produce, and every assertion built on+ * it would be measuring the fixture.*/+ function writtenObject(+ generation: number | undefined,+ resourceVersion: string,+ operationId: string | null = OPERATION_ID+ ) {+ const annotations: Record<string, string> = { [GENERATION]: '5' }+ if (operationId) annotations[INTENT_ID] = operationId+ return {+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion,+ ...(generation === undefined ? {} : { generation }),+ annotations,+ },+ spec: { contextRef: 'next' },+ }+ }++ /** The object as it stands before the write: live generation 4, no intent. */+ function liveObject() {+ return {+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion: '5',+ generation: 4,+ annotations: {},+ },+ spec: { contextRef: 'live' },+ }+ }+function api(replaced: Record<string, unknown>, patch = vi.fn().mockResolvedValue({})) {return {- getNamespacedCustomObject: vi.fn().mockResolvedValue({- metadata: {- name: 'host-a',- namespace: 'mcp-host',- resourceVersion: '5',- generation: 4,- annotations: {},- },- spec: { contextRef: 'live' },- }),+ getNamespacedCustomObject: vi.fn().mockResolvedValue(liveObject()),replaceNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),patchNamespacedCustomObject: patch,+ createNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),listNamespacedCustomObject: vi.fn(),}}- async function update(customApi: ReturnType<typeof api>) {+ function service(customApi: ReturnType<typeof api>) {+ return new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })+ }++ async function asOperator<T>(run: () => Promise<T>): Promise<T> {withIntent()- const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })try {return await runWithAdministrativeRequestContext({ operatorSub: 'admin-1', requestId: 'request-1' },- () =>- service.updateResource('hosts', 'host-a', { spec: { contextRef: 'next' } }, 'mcp-host')+ run)} finally {setAdministrativeOperationService(null)}}+ async function update(customApi: ReturnType<typeof api>) {+ return asOperator(() =>+ service(customApi).updateResource(+ 'hosts',+ 'host-a',+ { spec: { contextRef: 'next' } },+ 'mcp-host'+ )+ )+ }++ /**+ * Reads the reconcile-failure counter for one reason. Used as a liveness+ * witness that needs no mock: only the branch under test increments its own+ * reason, so a delta of 1 proves that exact branch ran. Counters are+ * process-global and other tests share them, hence the before/after delta+ * rather than an absolute value.+ */+ async function failureCount(reason: string): Promise<number> {+ const { values } = await governedTraceAdministrativeIntentReconcileFailedTotal.get()+ return values+ .filter(value => value.labels.reason === reason)+ .reduce((sum, value) => sum + value.value, 0)+ }+it('lowers the annotation to the persisted generation when the replace did not bump it', async () => {const patch = vi.fn().mockResolvedValue({})- const customApi = api(- { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },- patch- )+ const customApi = api(writtenObject(4, '6'), patch)await update(customApi)@@ -758,76 +815,225 @@ describe('ResourceService administrative intent generation reconciliation (#329)expect(patchArgs.body).not.toHaveProperty('spec')})- it('does not patch when the apiserver assigned the predicted generation', async () => {+ /**+ * A differential, because the equality branch emits nothing observable: no+ * patch, no log, no counter. "The patch did not fire" is therefore satisfied+ * just as well by a reconcile that was never called at all, so the matching+ * case is measured against a mismatching one driven through the SAME service+ * in the same test. The second half is the liveness witness for the first.+ *+ * Deleting the reconcile call entirely takes the second half red, which is+ * exactly the mutation this test previously survived.+ */+ it('patches on a mismatch and not on a match, with the correction armed both times', async () => {+ const matched = vi.fn().mockResolvedValue({})+ await update(api(writtenObject(5, '6'), matched))+ expect(matched).not.toHaveBeenCalled()++ const mismatched = vi.fn().mockResolvedValue({})+ await update(api(writtenObject(4, '6'), mismatched))+ expect(mismatched).toHaveBeenCalledOnce()+ expect(mismatched.mock.calls[0]![0].body.metadata.annotations[GENERATION]).toBe('4')+ })++ /**+ * Ownership, not generation, decides whether the correction is still ours.+ * The two annotation keys are written as a pair, so patching the generation+ * key while another operation owns the id key would fuse that operation's id+ * to our generation — a pair that never existed on any object.+ *+ * Same shape as the test above: the owned case is the witness for the+ * reassigned one, and the reassigned branch emits no counter because it is a+ * correct decision to skip, not a failure to apply.+ */+ it('skips the correction when another operation owns the intent annotation', async () => {+ const ours = vi.fn().mockResolvedValue({})+ await update(api(writtenObject(4, '6', OPERATION_ID), ours))+ expect(ours).toHaveBeenCalledOnce()++ const theirs = vi.fn().mockResolvedValue({})+ await update(api(writtenObject(4, '6', OTHER_OPERATION_ID), theirs))+ expect(theirs).not.toHaveBeenCalled()+ })++ /**+ * An object with no intent id carries no pending obligation, so writing the+ * generation key alone would leave half a pair naming no operation. Distinct+ * from reassignment, and counted under its own reason so the two cannot be+ * confused on a dashboard.+ */+ it('counts an absent intent annotation under its own reason and writes nothing', async () => {const patch = vi.fn().mockResolvedValue({})- const customApi = api(- { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 5, resourceVersion: '6' } },- patch- )+ const before = await failureCount('intent_absent')++ await update(api(writtenObject(4, '6', null), patch))++ // Witness: the counter this branch alone increments moved by exactly one.+ expect((await failureCount('intent_absent')) - before).toBe(1)+ expect(patch).not.toHaveBeenCalled()+ })++ /**+ * The object moved past the generation this operation produced. Correct it+ * ANYWAY: `persisted` is what this write actually produced, which stays true+ * wherever the object has since got to, and writing it lands+ * `annotated < live` — the terminal refusal the binding resolver exists to+ * make.+ *+ * Leaving the PREDICTION in place is the dangerous option. `predicted` is by+ * construction the next generation the object is most likely to reach, so a+ * concurrent bump can make it match the live generation exactly, and the+ * resolver would then bind another writer's change to this operator's intent+ * — the audit-trail falsification #329 exists to prevent.+ */+ it('corrects to the produced generation even when the object outpaced it', async () => {+ const patch = vi.fn().mockRejectedValueOnce(makeConflictError()).mockResolvedValue({})+ const customApi = api(writtenObject(4, '6'), patch)+ customApi.getNamespacedCustomObject.mockResolvedValueOnce(liveObject()).mockResolvedValue({+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion: '9',+ generation: 6,+ annotations: { [INTENT_ID]: OPERATION_ID, [GENERATION]: '5' },+ },+ spec: { contextRef: 'other' },+ })await update(customApi)- // Liveness witness for the negative assertion: the write really happened- // and really annotated 5, so the absent patch is the equality check and- // not an unexecuted path.- expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()- expect(- customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body.metadata.annotations[- GENERATION- ]- ).toBe('5')+ // The retry re-read and patched again — the loop's reason for existing,+ // untested before this. The second patch carries the generation THIS write+ // produced (4), below the live 6, against the re-read's resourceVersion.+ expect(patch).toHaveBeenCalledTimes(2)+ expect(patch.mock.calls[1]![0].body.metadata).toMatchObject({+ resourceVersion: '9',+ annotations: { [GENERATION]: '4' },+ })+ })++ /**+ * Three conflicts exhaust the loop. The write itself succeeded, so the method+ * still returns normally; the reason is reported rather than swallowed.+ */+ it('reports patch exhaustion after the last conflict instead of throwing', async () => {+ const patch = vi.fn().mockRejectedValue(makeConflictError())+ const customApi = api(writtenObject(4, '6'), patch)+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce(liveObject())+ .mockResolvedValue(writtenObject(4, '9'))+ const before = await failureCount('patch_failed')++ await expect(update(customApi)).resolves.toMatchObject({ metadata: { name: 'host-a' } })++ expect(patch).toHaveBeenCalledTimes(3)+ expect((await failureCount('patch_failed')) - before).toBe(1)+ })++ /**+ * Without a resourceVersion there is no optimistic-concurrency precondition,+ * and an unconditional patch could overwrite a concurrent write. Skipping is+ * the fail-closed answer; the counter says why.+ */+ it('refuses to patch without a resourceVersion precondition', async () => {+ const patch = vi.fn().mockResolvedValue({})+ const written = writtenObject(4, '6')+ delete (written.metadata as { resourceVersion?: string }).resourceVersion+ const before = await failureCount('no_precondition')++ await update(api(written, patch))++ expect((await failureCount('no_precondition')) - before).toBe(1)expect(patch).not.toHaveBeenCalled()})/**- * A 409 means the resourceVersion from the write is stale. The re-read- * decides whether the correction is still ours to make: if the object has- * moved past the generation THIS operation produced, writing either value- * would attribute a generation the intent did not cause, so the method stops- * and the binding resolver refuses those outcomes as drift — which is the- * correct answer, not a regression.+ * A create is the one prediction that cannot be wrong. `PrepareForCreate`+ * sets `metadata.generation = 1` unconditionally, and it runs AFTER mutating+ * admission, so no webhook can move it. The annotation is therefore written+ * as literal `1` and the correction never fires.+ *+ * The witness is the annotation on the create body: without it, "no patch"+ * would also describe a create that never annotated anything.*/- it('stops correcting when a concurrent write moved the object past our generation', async () => {- const conflict = makeConflictError()- const patch = vi.fn().mockRejectedValue(conflict)+ it('annotates a create with the literal generation 1 and corrects nothing', async () => {+ const patch = vi.fn().mockResolvedValue({})const customApi = api(- { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },- patch- )- // The re-read sees a generation past the one our replace produced.- customApi.getNamespacedCustomObject- .mockResolvedValueOnce({- metadata: {- name: 'host-a',- namespace: 'mcp-host',- resourceVersion: '5',- generation: 4,- annotations: {},- },- spec: { contextRef: 'live' },- })- .mockResolvedValue({+ {metadata: {name: 'host-a',namespace: 'mcp-host',- resourceVersion: '9',- generation: 6,- annotations: {},+ generation: 1,+ resourceVersion: '1',+ annotations: { [INTENT_ID]: OPERATION_ID, [GENERATION]: '1' },},- spec: { contextRef: 'other' },- })+ },+ patch+ )- await update(customApi)+ await asOperator(() =>+ service(customApi).createResource(+ 'hosts',+ { metadata: { name: 'host-a' }, spec: { contextRef: 'next' } },+ 'mcp-host'+ )+ )++ const createBody = customApi.createNamespacedCustomObject.mock.calls[0]![0].body+ expect(createBody.metadata.annotations[GENERATION]).toBe('1')+ expect(patch).not.toHaveBeenCalled()+ })++ /**+ * `mutateResource` is the likeliest producer of drift in the repository: it+ * issues a replace even when the mutation computes an identical spec, and the+ * Host CRD's status subresource means an unchanged spec does not advance the+ * generation. Covered here because a wrong argument at that call site would+ * otherwise ship green — the update path's tests say nothing about it.+ */+ it('corrects a mutateResource replace that did not advance the generation', async () => {+ const patch = vi.fn().mockResolvedValue({})+ const customApi = api(writtenObject(4, '6'), patch)- // Liveness: one patch WAS attempted and got the 409, so the single call is- // the stop rule and not a method that never tried.+ await asOperator(() =>+ service(customApi).mutateResource(+ 'hosts',+ 'host-a',+ () => ({ spec: { contextRef: 'live' } }),+ 'mcp-host'+ )+ )++ expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()expect(patch).toHaveBeenCalledOnce()- expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)+ expect(patch.mock.calls[0]![0].body.metadata.annotations[GENERATION]).toBe('4')+ })++ /**+ * A re-read that fails leaves the annotation at the prediction. The write+ * succeeded, so the method still returns normally and reports the reason.+ * `extractK8sStatus` reads the status off the wrapped error, which is why it+ * had to learn `httpStatus`: `getResource` rejects with `K8sNotFoundError`,+ * whose status lives on that field alone.+ */+ it('reports a failed re-read instead of throwing, and reads its status', async () => {+ const patch = vi.fn().mockRejectedValue(makeConflictError())+ const customApi = api(writtenObject(4, '6'), patch)+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce(liveObject())+ .mockRejectedValue(new K8sNotFoundError('hosts/host-a not found in namespace mcp-host'))+ const before = await failureCount('reread_failed')++ await expect(update(customApi)).resolves.toMatchObject({ metadata: { name: 'host-a' } })++ expect(patch).toHaveBeenCalledOnce()+ expect((await failureCount('reread_failed')) - before).toBe(1)})it('reports rather than throws when the write response carries no generation', async () => {const patch = vi.fn().mockResolvedValue({})- const customApi = api({ metadata: { name: 'host-a', namespace: 'mcp-host' } }, patch)+ const customApi = api(writtenObject(undefined, '6'), patch)+ const before = await failureCount('generation_missing')// The mutation itself succeeded, so it must still return normally: turning// an uncorrectable annotation into a caller-visible error would report a@@ -836,6 +1042,10 @@ describe('ResourceService administrative intent generation reconciliation (#329)metadata: { name: 'host-a' },})expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()+ // Witness the name promises: the branch REPORTED. Without this the test+ // passes with the whole reconcile call deleted, since "no patch went out"+ // was true before this feature existed.+ expect((await failureCount('generation_missing')) - before).toBe(1)expect(patch).not.toHaveBeenCalled()})})diff --git a/control-api/test/services.routeTracingSubmission.test.ts b/control-api/test/services.routeTracingSubmission.test.tsindex a2984946e..69d7ddc6d 100644--- a/control-api/test/services.routeTracingSubmission.test.ts+++ b/control-api/test/services.routeTracingSubmission.test.ts@@ -236,7 +236,15 @@ describe('route tracing submission facade', () => {// Liveness: the rejection path ran to completion — the event was counted// as rejected and the error clock was stamped.+ // Labels, not just the count: a regression that counted the refusal under+ // the wrong label would leave a bare call-count assertion green while the+ // dashboard built on it moved the event to another series.expect(rejected).toHaveBeenCalledOnce()+ expect(rejected).toHaveBeenCalledWith({+ family: 'administrative',+ source: 'host-context-controller',+ type: 'linked_outcome',+ })expect(lastError).toHaveBeenCalledWith({ scope: 'administrative', reason: 'event_rejected' },expect.any(Number)diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex a0a12dbe2..eba90ec51 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.347",+ "version": "0.1.348","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.347",+ "version": "0.1.348","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 2fbf1bd8d..ba549ad76 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.347",+ "version": "0.1.348","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/administrativeOutcomeReporter.test.ts b/host-context-controller/src/administrativeOutcomeReporter.test.tsindex 6b42c387f..e6821f867 100644--- a/host-context-controller/src/administrativeOutcomeReporter.test.ts+++ b/host-context-controller/src/administrativeOutcomeReporter.test.ts@@ -353,7 +353,10 @@ describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)* `administrative_intent_generation_drift` forever, and before this change* that answer was a plain retryable failure: 3 submits per enqueue, and* `onDrop` deleted the dedupe key so the next reconciler pass enqueued it- * again. Measured in dev at ~108 requests/hour for a single drifted Host.+ * again. Measured in dev at 109 requests/hour for a single drifted Host (327+ * refusals over 3 hours); the arithmetic for 3 enqueues × 3 submits per+ * 300-second reconciler cycle predicts 108, which is how the measurement was+ * tied to this loop rather than to some other source of 403s.** The terminal classification routes it to `onTerminal`, which PRESERVES the* key, so the re-enqueue is deduplicated instead of resubmitted. The bounddiff --git a/host-context-controller/src/reporterHttpFailure.test.ts b/host-context-controller/src/reporterHttpFailure.test.tsindex a0a2030d5..b6c066f11 100644--- a/host-context-controller/src/reporterHttpFailure.test.ts+++ b/host-context-controller/src/reporterHttpFailure.test.ts@@ -32,6 +32,14 @@ describe('throwForFailedSubmit', () => {expect(json).not.toHaveBeenCalled()})+ /**+ * The first two rows share status 409 (#329), which is what makes the `find`+ * on `code` decide the terminal result rather than merely confirm the only+ * candidate. Replacing that `find` with "the first entry of this status"+ * turns the drift row into `conflict`, and HCC would record a routine+ * operator outcome as an idempotency collision — so the table below is the+ * falsifying test for that lookup, not just a listing of the pairs.+ */it.each([[409, 'tracing_idempotency_conflict', 'conflict'],[409, 'administrative_intent_generation_drift', 'rejected'],@@ -44,23 +52,6 @@ describe('throwForFailedSubmit', () => {expect(error).toMatchObject({ result })})- /**- * Two codes now share status 409 (#329), so the `find` on `code` decides- * which terminal result applies rather than merely confirming the only- * candidate. A resolver that returned the first 409 entry would answer- * `conflict` for a drift refusal, and HCC would record a routine operator- * outcome as an idempotency collision.- */- it('picks the terminal result by code when two entries share status 409', async () => {- const conflict = await failure(409, async () => ({ code: 'tracing_idempotency_conflict' }))- const drift = await failure(409, async () => ({- code: 'administrative_intent_generation_drift',- }))-- expect(conflict).toMatchObject({ result: 'conflict' })- expect(drift).toMatchObject({ result: 'rejected' })- })-it.each([[400, 'tracing_idempotency_conflict'],[409, 'unsafe_tracing_input'],diff --git a/host-context-controller/src/reporterHttpFailure.ts b/host-context-controller/src/reporterHttpFailure.tsindex 043517ba2..483085bfa 100644--- a/host-context-controller/src/reporterHttpFailure.ts+++ b/host-context-controller/src/reporterHttpFailure.ts@@ -17,7 +17,11 @@ const log = hccLogger.child({ module: 'reporter-http-failure' })* different reason than the others: the request is well formed and the server* state is real, but the Host carries an intent annotation pinned to a* generation it has already passed, and nothing ever retires that annotation.- * Generations only grow and the reporter always reads the live object, so the+ * Generations only grow — but on its own that would not settle it, because a+ * generation the annotation could still catch up to is not a superseded one.+ * The second fact is what closes it: nothing ever RAISES the annotation.+ * control-api's only correction lowers a prediction to the generation its write+ * actually produced, and the reporter always reads the live object, so the* answer cannot change. The plain 403 next door is NOT deterministic — it also* covers a binding that is merely not visible yet, which is what lets a* control-api/HCC deploy overlap heal itself — so only the coded 409 is listed