Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(tracing): retire drifted administrative intent annotations (#329)
Alfredo Lopeza90b619b61e1
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 198596027..6ae781fe2 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.593",+ "version": "0.1.594","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.593",+ "version": "0.1.594","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex 68e2a3c02..5b35eb47d 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.593",+ "version": "0.1.594","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/http/errorHandler.ts b/control-api/src/http/errorHandler.tsindex 25f61b94a..876f2f263 100644--- a/control-api/src/http/errorHandler.ts+++ b/control-api/src/http/errorHandler.ts@@ -20,6 +20,11 @@ export const FORWARDABLE_CLIENT_ERROR_CODES = new Set(['tracing_idempotency_conflict','unsafe_tracing_input','invalid_tracing_input',+ // Deterministic: the annotated generation is already superseded, so no+ // future report can bind. `tracing_binding_unavailable` stays OUT on+ // purpose — that 403 must remain code-less and therefore retryable, which+ // is what lets a control-api/HCC deploy overlap heal itself.+ 'administrative_intent_generation_drift',])/**diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.tsindex 4ad66acfa..170ba8483 100644--- a/control-api/src/observability/metrics.ts+++ b/control-api/src/observability/metrics.ts@@ -406,6 +406,22 @@ export const governedTraceConflictingTotal = getOrCreateCounter({labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,})+/**+ * A Host whose administrative-intent annotation names a generation the object+ * has already passed, so no outcome for it can ever be attributed (#329). The+ * authoritative drift signal: HCC's own reporter counter labels this+ * `result="rejected"`, the same label it uses for malformed input, and cannot+ * tell the two apart.+ *+ * `namespace` is the only label. Host names are unbounded, so putting one in a+ * label would make the series count grow with the cluster.+ */+export const governedTraceAdministrativeIntentDriftTotal = getOrCreateCounter({+ name: 'governed_trace_administrative_intent_drift_total',+ help: 'Count of administrative tracing events refused because the intent annotation names a superseded generation.',+ labelNames: ['namespace'] as const as Array<'namespace'>,+})+export const governedTraceIngestDurationSeconds = getOrCreateHistogram({name: 'governed_trace_ingest_duration_seconds',help: 'Duration of governed trace ingestion and stream registration.',diff --git a/control-api/src/services/resourceService.ts b/control-api/src/services/resourceService.tsindex ccb26a7ef..ca25ca2cb 100644--- a/control-api/src/services/resourceService.ts+++ b/control-api/src/services/resourceService.ts@@ -1,4 +1,5 @@import * as k8s from '@kubernetes/client-node'+import { rootLogger } from '../observability/logger.js'import {CLERUM_GROUP,CLERUM_VERSION,@@ -12,6 +13,7 @@ import {kindFromPlural,parseProjectedGeneration,} from './resourceServiceHelpers.js'+import { ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION } from './tracing/adminOperationConstants.js'import { currentAdministrativeRequestContext } from './tracing/adminOperationContext.js'import {ControlApiAdministrativeOperationService,@@ -253,8 +255,16 @@ export class ResourceService {name: body.metadata.name,})const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata) ?? body.metadata+ // A create is the one prediction that is almost always right — the+ // apiserver assigns generation 1 — but it is still a prediction, and an+ // admission webhook that mutates the spec can land the object at 2.+ const predictedGeneration = 1const annotations = intent- ? withAdministrativeIntentAnnotation(sanitizedMetadata.annotations, intent, 1)+ ? withAdministrativeIntentAnnotation(+ sanitizedMetadata.annotations,+ intent,+ predictedGeneration+ ): sanitizedMetadata.annotationsconst resource: ClerumResource = {apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,@@ -268,8 +278,9 @@ export class ResourceService {spec: body.spec,}+ let created: unknowntry {- return await this.customApi.createNamespacedCustomObject({+ created = await this.customApi.createNamespacedCustomObject({group: CLERUM_GROUP,version: CLERUM_VERSION,namespace: ns,@@ -280,6 +291,18 @@ export class ResourceService {await persistHostFailure(intent)throw err}+ // Outside the try on purpose: the create already succeeded, and a failure+ // to correct the annotation must not reach `persistHostFailure`, which+ // would record a completed operation as failed.+ await this.reconcileAdministrativeIntentGeneration({+ plural,+ name: body.metadata.name,+ namespace: ns,+ intent,+ predictedGeneration,+ written: created,+ })+ return created}async updateResource(@@ -343,12 +366,15 @@ export class ResourceService {current.metadata?.annotations,sanitizedMetadata?.annotations)+ // Predicted, not persisted: the annotation travels inside the same body+ // as the spec, so the generation the apiserver will assign is not+ // knowable yet. `reconcileAdministrativeIntentGeneration` corrects it+ // against the response — a replace whose spec is unchanged leaves the+ // generation where it was, and the prediction would pin the intent to a+ // generation that never arrives.+ const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)const annotations = intent- ? withAdministrativeIntentAnnotation(- mergedAnnotations,- intent,- Math.max(1, (current.metadata?.generation ?? 0) + 1)- )+ ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration): mergedAnnotationsconst resource: ClerumResource = {apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,@@ -363,8 +389,9 @@ export class ResourceService {spec: body.spec,}+ let replaced: unknowntry {- return await this.customApi.replaceNamespacedCustomObject({+ replaced = await this.customApi.replaceNamespacedCustomObject({group: CLERUM_GROUP,version: CLERUM_VERSION,namespace: ns,@@ -393,10 +420,177 @@ export class ResourceService {await persistHostFailure(intent)throw err}+ // After the catch, not inside the try: the replace already succeeded, so+ // a failure here must not reach `persistHostFailure` and record a+ // completed operation as failed.+ await this.reconcileAdministrativeIntentGeneration({+ plural,+ name,+ namespace: ns,+ intent,+ predictedGeneration,+ written: replaced,+ })+ return replaced}throw new Error(`Failed to update ${plural}/${name} after ${maxAttempts} attempts`)}+ /**+ * Replace the PREDICTED generation in the administrative-intent annotation+ * with the one the apiserver actually persisted (#329).+ *+ * The three write paths above annotate with `current.generation + 1` before+ * the write, because the annotation has to travel inside the same body. That+ * prediction is wrong whenever the write does not bump the generation, and+ * the Host CRD declares a status subresource+ * (`charts/clerum-crds/crds/host.yaml`), so `metadata.generation` advances+ * ONLY on spec changes. A replace whose spec is byte-identical — an+ * idempotent operator PUT, a `mutateResource` whose mutation is a no-op, a+ * hook-ref resync — leaves the object at N while the annotation says N+1.+ * Nothing ever retires that annotation, so the Host stops binding outcomes+ * forever.+ *+ * A merge-patch of annotations alone does not bump the generation, so this+ * correction cannot feed itself.+ *+ * NOT `patchAnnotationMonotonic`: that method takes the max of projected and+ * requested and refuses to lower a value. Lowering N+1 back to N is exactly+ * the correction needed here.+ *+ * Never throws. The write it corrects already succeeded, and turning a+ * successful mutation into a failed request would misreport what happened to+ * the object. Everything it cannot do is logged with the reason.+ */+ private async reconcileAdministrativeIntentGeneration(input: {+ plural: ClerumResourceType+ name: string+ namespace: string+ intent: HostAdministrativeIntent | null+ predictedGeneration: number+ written: unknown+ }): Promise<void> {+ if (!input.intent) return+ const metadata = (input.written as { metadata?: { generation?: number } } | null)?.metadata+ const persisted = metadata?.generation+ if (typeof persisted !== 'number' || !Number.isSafeInteger(persisted)) {+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_unverified',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ predictedGeneration: input.predictedGeneration,+ },+ 'write response carried no usable metadata.generation; the intent annotation was left at the predicted value'+ )+ return+ }+ if (persisted === input.predictedGeneration) return++ const maxAttempts = 3+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {+ // The first attempt uses the write's own response; a retry re-reads,+ // because a 409 means the resourceVersion we held is stale.+ let snapshot: { metadata?: { generation?: number; resourceVersion?: string } }+ if (attempt === 1) {+ snapshot = input.written as { metadata?: { generation?: number; resourceVersion?: string } }+ } else {+ try {+ snapshot = (await this.getResource(input.plural, input.name, input.namespace)) as {+ metadata?: { generation?: number; resourceVersion?: string }+ }+ } catch (err) {+ // The only place in this method that can throw. Caught so the+ // documented contract holds, and reported rather than swallowed.+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_patch_failed',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ predictedGeneration: input.predictedGeneration,+ persistedGeneration: persisted,+ status: extractK8sStatus(err),+ },+ 're-read failed while correcting the intent annotation to the persisted generation'+ )+ return+ }+ }+ if (snapshot.metadata?.generation !== persisted) {+ // Something else advanced the object past the generation this+ // operation produced. Writing `persisted` now would attribute a+ // generation this intent did not cause; writing the newer one would+ // do the same. The intent is genuinely unattributable from here, and+ // the binding resolver refuses its outcomes by design (#329).+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_superseded',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ writtenGeneration: persisted,+ liveGeneration: snapshot.metadata?.generation,+ },+ 'object advanced past the generation this operation produced; intent annotation left uncorrected'+ )+ return+ }+ const resourceVersion = snapshot.metadata?.resourceVersion+ if (!resourceVersion) {+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_unverified',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ predictedGeneration: input.predictedGeneration,+ persistedGeneration: persisted,+ },+ 'no metadata.resourceVersion to use as a patch precondition; the intent annotation was left at the predicted value'+ )+ return+ }+ try {+ await this.customApi.patchNamespacedCustomObject(+ {+ group: CLERUM_GROUP,+ version: CLERUM_VERSION,+ namespace: input.namespace,+ plural: input.plural,+ name: input.name,+ body: {+ metadata: {+ resourceVersion,+ annotations: {+ [ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION]: String(persisted),+ },+ },+ },+ },+ k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)+ )+ return+ } catch (err) {+ if (extractK8sStatus(err) === 409 && attempt < maxAttempts) continue+ rootLogger.warn(+ {+ event: 'administrative_intent_generation_patch_failed',+ plural: input.plural,+ namespace: input.namespace,+ name: input.name,+ predictedGeneration: input.predictedGeneration,+ persistedGeneration: persisted,+ status: extractK8sStatus(err),+ },+ 'could not correct the intent annotation to the persisted generation'+ )+ return+ }+ }+ }+/*** Merge-patch ONLY metadata.annotations of a custom object.*@@ -563,12 +757,15 @@ export class ResourceService {current.metadata?.annotations,sanitizedMetadata?.annotations)+ // Predicted, not persisted. This path is the likeliest producer of drift+ // in the repository: it does not short-circuit on an unchanged spec (the+ // `if (!next)` above only fires when the mutation returns nothing), so a+ // mutation that computes the same spec still issues a replace, and the+ // Host CRD's status subresource means such a replace leaves the+ // generation untouched while the prediction moves ahead of it.+ const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)const annotations = intent- ? withAdministrativeIntentAnnotation(- mergedAnnotations,- intent,- Math.max(1, (current.metadata?.generation ?? 0) + 1)- )+ ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration): mergedAnnotationsconst resource: ClerumResource = {apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,@@ -583,8 +780,9 @@ export class ResourceService {spec: next.spec,}+ let replaced: unknowntry {- return await this.customApi.replaceNamespacedCustomObject({+ replaced = await this.customApi.replaceNamespacedCustomObject({group: CLERUM_GROUP,version: CLERUM_VERSION,namespace: ns,@@ -605,6 +803,15 @@ export class ResourceService {await persistHostFailure(intent)throw err}+ await this.reconcileAdministrativeIntentGeneration({+ plural,+ name,+ namespace: ns,+ intent,+ predictedGeneration,+ written: replaced,+ })+ return replaced}throw new Error(`Failed to mutate ${plural}/${name} after ${maxAttempts} attempts`)}diff --git a/control-api/src/services/tracing/adminOperationBindingResolver.ts b/control-api/src/services/tracing/adminOperationBindingResolver.tsindex 026a9c4ee..9bd75c6ec 100644--- a/control-api/src/services/tracing/adminOperationBindingResolver.ts+++ b/control-api/src/services/tracing/adminOperationBindingResolver.ts@@ -1,4 +1,6 @@import type { AdministrativeEventSubmitterPrincipalV1 } from '../../middleware/tracingSubmitterAuth.js'+import { type Logger, rootLogger } from '../../observability/logger.js'+import { governedTraceAdministrativeIntentDriftTotal } from '../../observability/metrics.js'import {ADMINISTRATIVE_INTENT_ANNOTATION,ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION,@@ -8,7 +10,11 @@ import {administrativeIntentLookupKey,} from './adminOperationService.js'import type { AdministrativeEventInputV1, AdministrativeServerBindingV1 } from './contracts.js'-import type { AdministrativeOperationBindingResolver } from './routeSubmissionService.js'+import {+ type AdministrativeOperationBindingResolver,+ type BindingRefusal,+ isBindingRefusal,+} from './routeSubmissionService.js'export interface HostAdministrativeLookup {getResource(plural: 'hosts', name: string, namespace: string): Promise<unknown>@@ -45,23 +51,43 @@ const STATUS_REF =/^host:([a-z0-9]([-a-z0-9]*[a-z0-9])?)\/([a-z0-9]([-a-z0-9]*[a-z0-9])?):generation=([1-9][0-9]*):uid=([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i+const DRIFT_REFUSAL: BindingRefusal = { refusal: 'administrative_intent_generation_drift' }++type Candidate = {+ reference: { namespace: string; name: string; generation: number; uid: string }+ operationId: string+ targetRef: string+ outcome: 'succeeded' | 'failed'+}++/** The one counter method this resolver uses, so a test can pass a fake. */+export type DriftCounter = { inc(labels: { namespace: string }): void }+export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOperationBindingResolver {+ private readonly logger: Logger+ private readonly metrics: DriftCounter+constructor(private readonly hostLookup: HostAdministrativeLookup,- private readonly intentLookup: AdministrativeIntentLookup- ) {}+ private readonly intentLookup: AdministrativeIntentLookup,+ logger: Logger = rootLogger,+ metrics: DriftCounter = governedTraceAdministrativeIntentDriftTotal+ ) {+ this.logger = logger+ this.metrics = metrics+ }async resolve(principal: AdministrativeEventSubmitterPrincipalV1,event: AdministrativeEventInputV1- ): Promise<AdministrativeServerBindingV1 | null> {+ ): Promise<AdministrativeServerBindingV1 | BindingRefusal | null> {return (await this.resolveMany(principal, [event]))[0] ?? null}async resolveMany(principal: AdministrativeEventSubmitterPrincipalV1,events: readonly AdministrativeEventInputV1[]- ): Promise<readonly (AdministrativeServerBindingV1 | null)[]> {+ ): Promise<readonly (AdministrativeServerBindingV1 | BindingRefusal | null)[]> {if (principal.kind !== 'hcc_internal_control') {return events.map(() => null)}@@ -105,25 +131,49 @@ export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOpmetadata.annotations?.[ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION])const targetRef = `${reference.namespace}/${reference.name}`- if (- !operationId ||- !UUID.test(operationId) ||- !Number.isSafeInteger(expectedGeneration) ||- expectedGeneration !== reference.generation- )+ if (!operationId || !UUID.test(operationId) || !Number.isSafeInteger(expectedGeneration))return null+ if (expectedGeneration < reference.generation) {+ // Every identity field above already matched, so the live object IS the+ // one the reporter observed, and its generation has moved past the one+ // the annotation pins. Generations only grow and nothing ever retires+ // the annotation, so this pair can never bind again: terminal, not+ // "unavailable right now".+ this.metrics.inc({ namespace: reference.namespace })+ // The Host name goes to the log, never to the metric label: names are+ // unbounded and a label carrying one grows the series count with the+ // cluster. The operationId is omitted from both.+ this.logger.warn(+ {+ event: 'administrative_intent_generation_drift',+ namespace: reference.namespace,+ name: reference.name,+ liveGeneration: reference.generation,+ annotatedGeneration: expectedGeneration,+ },+ 'administrative intent annotation is pinned to a superseded generation'+ )+ return DRIFT_REFUSAL+ }+ // `expectedGeneration > reference.generation` stays retryable: that is the+ // window between control-api predicting a generation and the corrective+ // annotation patch landing, and it resolves on its own.+ if (expectedGeneration !== reference.generation) return nullconst outcome = safeOutcome(events[index]!.payload?.status)if (!outcome) return nullreturn { reference, operationId, targetRef, outcome }})- const intentInputs = candidates.filter(Boolean).map(candidate => ({- operationId: candidate!.operationId,- targetRef: candidate!.targetRef,- namespace: candidate!.reference.namespace,- }))+ const intentInputs = candidates+ .filter(candidate => candidate !== null && !isBindingRefusal(candidate))+ .map(candidate => ({+ operationId: (candidate as Candidate).operationId,+ targetRef: (candidate as Candidate).targetRef,+ namespace: (candidate as Candidate).reference.namespace,+ }))const intents = await this.intentLookup.findHostIntents(intentInputs)return candidates.map(candidate => {if (!candidate) return null+ if (isBindingRefusal(candidate)) return candidateconst intentInput = {operationId: candidate.operationId,targetRef: candidate.targetRef,diff --git a/control-api/src/services/tracing/routeSubmissionService.ts b/control-api/src/services/tracing/routeSubmissionService.tsindex 1049a5bc0..f3ecab482 100644--- a/control-api/src/services/tracing/routeSubmissionService.ts+++ b/control-api/src/services/tracing/routeSubmissionService.ts@@ -62,15 +62,37 @@ export interface AgentRunBindingResolver {): MaybePromise<readonly (AgentRunServerBindingV1 | null | undefined)[]>}+/**+ * A resolver's way of saying "this event can never bind", as opposed to `null`,+ * which means "it cannot bind right now". The two need different HTTP statuses+ * because HCC retries one and gives up on the other.+ *+ * It is a value rather than a throw so the resolver can answer per event while+ * `resolveTrustedBindings` keeps the event's index — the same reason `null` is+ * a value there. It does NOT mean the batch continues: the first unbindable+ * event still aborts the whole submission (see `resolveTrustedBindings`).+ */+export type BindingRefusal = {+ readonly refusal: 'administrative_intent_generation_drift'+}++export function isBindingRefusal(value: unknown): value is BindingRefusal {+ return (+ typeof value === 'object' &&+ value !== null &&+ (value as { refusal?: unknown }).refusal === 'administrative_intent_generation_drift'+ )+}+export interface AdministrativeOperationBindingResolver {resolve(principal: AdministrativeEventSubmitterPrincipalV1,event: AdministrativeEventInputV1- ): MaybePromise<AdministrativeServerBindingV1 | null | undefined>+ ): MaybePromise<AdministrativeServerBindingV1 | BindingRefusal | null | undefined>resolveMany?(principal: AdministrativeEventSubmitterPrincipalV1,events: readonly AdministrativeEventInputV1[]- ): MaybePromise<readonly (AdministrativeServerBindingV1 | null | undefined)[]>+ ): MaybePromise<readonly (AdministrativeServerBindingV1 | BindingRefusal | null | undefined)[]>}export interface InfrastructureWorkloadBindingResolver {@@ -85,11 +107,14 @@ export interface InfrastructureWorkloadBindingResolver {}type BindingResolver<Principal, Event, Binding> = {- resolve(principal: Principal, event: Event): MaybePromise<Binding | null | undefined>+ resolve(+ principal: Principal,+ event: Event+ ): MaybePromise<Binding | BindingRefusal | null | undefined>resolveMany?(principal: Principal,events: readonly Event[]- ): MaybePromise<readonly (Binding | null | undefined)[]>+ ): MaybePromise<readonly (Binding | BindingRefusal | null | undefined)[]>}export class TracingBindingUnavailableError extends Error {@@ -106,6 +131,30 @@ export class TracingBindingUnavailableError extends Error {}}+/**+ * The reported object carries an administrative-intent annotation pinned to a+ * generation the object has already passed, so no future report can ever bind:+ * the reporter always reads the live object and `metadata.generation` only+ * grows. Deterministic by construction, which is why it is a 409 with a+ * forwarded code and not the retryable 403 that `TracingBindingUnavailableError`+ * produces.+ *+ * The message carries no Host identity: `name`/`namespace`/`uid` go to the+ * structured log, never to a response body.+ */+export class AdministrativeIntentGenerationDriftError extends Error {+ readonly code = 'administrative_intent_generation_drift'+ readonly status = 409+ readonly statusCode = 409++ constructor(readonly eventIndex: number) {+ super(+ `administrative intent annotation is pinned to a superseded generation for tracing event ${eventIndex}`+ )+ this.name = 'AdministrativeIntentGenerationDriftError'+ }+}+export class InvalidTracingInputError extends TypeError {readonly code = 'invalid_tracing_input'readonly status = 400@@ -375,7 +424,12 @@ async function resolveTrustedBindings<Principal, Event, Binding>(input: {`trusted ${input.bindingKind} binding resolver returned an unexpected batch size`)}+ // All-or-nothing, unchanged: the first event that does not resolve throws+ // before any append runs, so a batch holding one drifted event is refused+ // whole and nothing is stored. A refusal only picks a different status than+ // `null` does; it does not let the batch continue.return resolved.map((binding, index) => {+ if (isBindingRefusal(binding)) throw new AdministrativeIntentGenerationDriftError(index)if (!binding) throw new TracingBindingUnavailableError(input.bindingKind, index)return binding})@@ -444,6 +498,11 @@ function rejectionReason(error instanceof InvalidTracingInputError ||error instanceof UnsafeTracingInputError ||error instanceof TracingBindingUnavailableError ||+ // A 409 that is not an idempotency conflict falls past the status check+ // below (which only lists 400 and 403) into `submission_failed`, which+ // operationalStatus treats as critical. Naming it here keeps a drifted+ // Host an event-level rejection instead of a critical service alert.+ error instanceof AdministrativeIntentGenerationDriftError ||(error !== null &&typeof error === 'object' &&('status' in error || 'statusCode' in error) &&diff --git a/control-api/test/http.errorHandler.clientErrorCode.test.ts b/control-api/test/http.errorHandler.clientErrorCode.test.tsindex 09731b17d..2db8129b3 100644--- a/control-api/test/http.errorHandler.clientErrorCode.test.ts+++ b/control-api/test/http.errorHandler.clientErrorCode.test.ts@@ -8,6 +8,7 @@ import {} from '../src/services/tracing/append.js'import { GovernedReadInvalidQueryError } from '../src/services/tracing/governedEventReadService.js'import {+ AdministrativeIntentGenerationDriftError,InvalidTracingInputError,TracingBindingUnavailableError,} from '../src/services/tracing/routeSubmissionService.js'@@ -33,6 +34,11 @@ describe('clerumErrorHandler — machine-readable code on allowlisted 4xx', () =],[new UnsafeTracingInputError('payload.gfs_subject'), 400, 'unsafe_tracing_input'],[new InvalidTracingInputError('events[0].kind is not supported'), 400, 'invalid_tracing_input'],+ [+ new AdministrativeIntentGenerationDriftError(0),+ 409,+ 'administrative_intent_generation_drift',+ ],])('forwards %s with its code', async (err, status, code) => {const res = await request(appThrowing(err)).post('/boom')diff --git a/control-api/test/routes.tracingSubmissionBoundary.test.ts b/control-api/test/routes.tracingSubmissionBoundary.test.tsindex 56d378dc9..7f37bc2f0 100644--- a/control-api/test/routes.tracingSubmissionBoundary.test.ts+++ b/control-api/test/routes.tracingSubmissionBoundary.test.ts@@ -464,49 +464,61 @@ describe('internal tracing submission routers — rejection code on the wire', (* proves the handler forwards the error it is handed; this one has to reach* the real resolver through the real route.*/- it('answers an administrative outcome with a pre-uid sourceStatusRef 403 through the real route', async () => {- const OPERATION_ID = '11111111-1111-4111-8111-111111111111'- const listResource = vi.fn().mockResolvedValue([- {- apiVersion: 'clerum.io/v1alpha1',- kind: 'Host',- metadata: {- name: 'chatllm',- namespace: 'mcp-host',- generation: 7,- uid: HOST_UID,- annotations: {- 'clerum.io/administrative-intent-id': OPERATION_ID,- 'clerum.io/administrative-intent-generation': '7',- },+ const OPERATION_ID = '11111111-1111-4111-8111-111111111111'++ function hostResource(name: string, generation: number, intentGeneration: number) {+ return {+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'Host',+ metadata: {+ name,+ namespace: 'mcp-host',+ generation,+ uid: HOST_UID,+ annotations: {+ 'clerum.io/administrative-intent-id': OPERATION_ID,+ 'clerum.io/administrative-intent-generation': String(intentGeneration),},},- ])- const findHostIntents = vi.fn().mockResolvedValue(- new Map([- [- administrativeIntentLookupKey({- operationId: OPERATION_ID,- targetRef: 'mcp-host/chatllm',- namespace: 'mcp-host',- }),- {- operatorSub: 'admin-1',- requestId: 'request-1',- environment: 'test',- tenantId: null,- teamId: null,- identityIssuer: 'control-api',- operatorUserId: '22222222-2222-4222-8222-222222222222',- resourceAud: 'control-ui',- effectiveScopes: [],- tokenExchangeId: null,- authorizationDecision: 'allow' as const,- decisionActorSub: 'control-api',- },- ],+ }+ }++ function intentFor(...names: readonly string[]) {+ return new Map(+ names.map(name => [+ administrativeIntentLookupKey({+ operationId: OPERATION_ID,+ targetRef: `mcp-host/${name}`,+ namespace: 'mcp-host',+ }),+ {+ operatorSub: 'admin-1',+ requestId: 'request-1',+ environment: 'test',+ tenantId: null,+ teamId: null,+ identityIssuer: 'control-api',+ operatorUserId: '22222222-2222-4222-8222-222222222222',+ resourceAud: 'control-ui',+ effectiveScopes: [],+ tokenExchangeId: null,+ authorizationDecision: 'allow' as const,+ decisionActorSub: 'control-api',+ },]))+ }++ /**+ * The real administrative router over the real submission service, the real+ * HCC resolver and the global handler. Only the API-server list and the+ * database append are doubles.+ */+ function realAdministrativeApp(hosts: ReadonlyArray<ReturnType<typeof hostResource>>) {+ const listResource = vi.fn().mockResolvedValue([...hosts])+ const findHostIntents = vi+ .fn()+ .mockResolvedValue(intentFor(...hosts.map(host => host.metadata.name)))const appendManyInTransaction = vi.fn().mockResolvedValue([{kind: 'accepted' as const,@@ -535,23 +547,28 @@ describe('internal tracing submission routers — rejection code on the wire', ())app.use(clerumErrorHandler)+ return { app, listResource, appendManyInTransaction }+ }- const post = (sourceStatusRef: string) =>- request(app)- .post('/internal/tracing/administrative-events')- .set('Authorization', `Bearer ${signInternalControl('hcc')}`)- .send({- events: [- {- kind: 'linked_outcome',- sourceEventId: `outcome-${sourceStatusRef.length}`,- occurredAt: '2026-07-10T09:59:59.000Z',- reasonCode: 'boundary_test',- sourceStatusRef,- payload: { resource_class: 'Host', status: 'succeeded' },- },- ],- })+ function postOutcomes(app: express.Express, refs: readonly string[]) {+ return request(app)+ .post('/internal/tracing/administrative-events')+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)+ .send({+ events: refs.map((sourceStatusRef, index) => ({+ kind: 'linked_outcome',+ sourceEventId: `outcome-${index}-${sourceStatusRef.length}`,+ occurredAt: '2026-07-10T09:59:59.000Z',+ reasonCode: 'boundary_test',+ sourceStatusRef,+ payload: { resource_class: 'Host', status: 'succeeded' },+ })),+ })+ }++ it('answers an administrative outcome with a pre-uid sourceStatusRef 403 through the real route', async () => {+ const { app, appendManyInTransaction } = realAdministrativeApp([hostResource('chatllm', 7, 7)])+ const post = (sourceStatusRef: string) => postOutcomes(app, [sourceStatusRef])// Liveness: the current format binds and stores through this same app, so// the refusal below is the legacy format and not a route that refuses all.@@ -586,4 +603,78 @@ describe('internal tracing submission routers — rejection code on the wire', (expect(response.body.correlationId).toEqual(expect.any(String))expect(response.body).not.toHaveProperty('code')})++ /**+ * #329 through the real route. The unit test proves the resolver returns a+ * refusal; only this proves the refusal survives the submission service, the+ * router and the global handler as a 409 carrying its code — which is the+ * only form HCC can classify as terminal.+ */+ describe('administrative intent generation drift (#329)', () => {+ const LIVE_REF = `host:mcp-host/chatllm:generation=7:uid=${HOST_UID}`+ const DRIFTED_REF = `host:mcp-host/drifted:generation=7:uid=${HOST_UID}`++ it('answers a superseded intent annotation 409 with its code through the real route', async () => {+ const { app, appendManyInTransaction } = realAdministrativeApp([+ hostResource('chatllm', 7, 7),+ hostResource('drifted', 7, 6),+ ])++ // Liveness: the healthy Host binds and stores through this same app, so+ // the 409 below is the drifted annotation and not a route that refuses+ // everything.+ const accepted = await postOutcomes(app, [LIVE_REF])+ expect(accepted.status).toBe(200)+ expect(appendManyInTransaction).toHaveBeenCalledOnce()++ const refused = await postOutcomes(app, [DRIFTED_REF])++ expect(refused.status).toBe(409)+ expect(refused.body).toMatchObject({+ code: 'administrative_intent_generation_drift',+ correlationId: expect.any(String),+ })+ // Nothing was appended for the refused request: the count is still the+ // one call the liveness probe made.+ expect(appendManyInTransaction).toHaveBeenCalledOnce()+ })++ it('keeps Host identity out of the refusal body', async () => {+ const { app } = realAdministrativeApp([hostResource('drifted', 7, 6)])++ const refused = await postOutcomes(app, [DRIFTED_REF])++ expect(refused.status).toBe(409)+ // Liveness: the refusal really is the drift branch, so the absence of+ // the name below is not an unrelated failure with an empty body.+ expect(refused.body.code).toBe('administrative_intent_generation_drift')+ const serialized = JSON.stringify(refused.body)+ expect(serialized).not.toContain('drifted')+ expect(serialized).not.toContain('mcp-host')+ expect(serialized).not.toContain(HOST_UID)+ })++ /**+ * `resolveTrustedBindings` throws on the first event it cannot resolve,+ * before any append runs, so a batch holding one drifted event is refused+ * whole. The refusal changes which status the batch gets; it does not let+ * the batch continue past the bad event.+ */+ it('refuses a batch whole when one of its events has drifted', async () => {+ const { app, listResource, appendManyInTransaction } = realAdministrativeApp([+ hostResource('chatllm', 7, 7),+ hostResource('drifted', 7, 6),+ ])++ const mixed = await postOutcomes(app, [LIVE_REF, DRIFTED_REF])++ expect(mixed.status).toBe(409)+ expect(mixed.body.code).toBe('administrative_intent_generation_drift')+ // Liveness: the resolver did run over the whole batch — it read the+ // Hosts — so "nothing stored" is the all-or-nothing rule and not a+ // request that never reached the service.+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')+ expect(appendManyInTransaction).not.toHaveBeenCalled()+ })+ })})diff --git a/control-api/test/services.adminOperationBindingResolver.test.ts b/control-api/test/services.adminOperationBindingResolver.test.tsindex dc44669bb..5c68563ac 100644--- a/control-api/test/services.adminOperationBindingResolver.test.ts+++ b/control-api/test/services.adminOperationBindingResolver.test.ts@@ -124,7 +124,10 @@ describe('HccAdministrativeOutcomeBindingResolver', () => {it.each([['stale generation', host(undefined, 8), true],- ['stale intent annotation', host(undefined, 7, 6), true],+ // `host(undefined, 7, 6)` — the annotation pinned BEHIND the live+ // generation — used to belong here. It is now a refusal, not a null, and+ // has its own block below (#329).+ ['an intent annotation ahead of the live generation', host(undefined, 7, 8), true],['invalid annotation', host('caller-value'), true],['missing intent', host(), false],// A Host deleted and recreated under the same name reaches the same@@ -215,4 +218,110 @@ describe('HccAdministrativeOutcomeBindingResolver', () => {expect(malformed).toBeNull()expect(witness).toMatchObject({ targetRef: 'mcp-host/chatllm-live', outcome: 'succeeded' })})++ /**+ * #329 — the annotation is an obligation nothing ever retires, so once the+ * Host passes the generation it names, no future report can bind. That is a+ * different answer from "not yet visible", and the two must not collapse+ * into the same null.+ */+ describe('administrative intent generation drift (#329)', () => {+ it('refuses an annotation pinned behind the live generation, and counts it', async () => {+ const metrics = { inc: vi.fn() }+ const warn = vi.fn()+ const listResource = vi.fn().mockResolvedValue([host(undefined, 7, 6)])+ const findHostIntents = vi.fn().mockResolvedValue(intentMap())++ const result = await new HccAdministrativeOutcomeBindingResolver(+ { getResource: vi.fn(), listResource },+ { findHostIntent: vi.fn(), findHostIntents },+ { warn } as never,+ metrics+ ).resolve(principal, event)++ expect(result).toEqual({ refusal: 'administrative_intent_generation_drift' })+ // Liveness witnesses: the refusal is not the resolver skipping the+ // candidate loop. It read the Hosts, it reached the decision, and it+ // reported the decision.+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')+ expect(metrics.inc).toHaveBeenCalledWith({ namespace: 'mcp-host' })+ expect(warn).toHaveBeenCalledWith(+ expect.objectContaining({+ event: 'administrative_intent_generation_drift',+ namespace: 'mcp-host',+ liveGeneration: 7,+ annotatedGeneration: 6,+ }),+ expect.any(String)+ )+ })++ /**+ * The asymmetry is the whole point. `annotated < live` can never bind+ * again; `annotated > live` is the window between control-api predicting a+ * generation and the corrective patch landing, and it resolves on its own.+ * Both rows run through the SAME call, so a resolver that answered one way+ * for everything would fail.+ */+ it('separates a superseded annotation from one that is merely ahead', async () => {+ const metrics = { inc: vi.fn() }+ const behind = host(undefined, 7, 6)+ const ahead = host(undefined, 7, 9)+ ahead.metadata.name = 'chatllm-ahead'++ const resolver = new HccAdministrativeOutcomeBindingResolver(+ { getResource: vi.fn(), listResource: vi.fn().mockResolvedValue([behind, ahead]) },+ {+ findHostIntent: vi.fn(),+ findHostIntents: vi+ .fn()+ .mockResolvedValue(new Map([...intentMap(), ...intentMap(undefined, 'chatllm-ahead')])),+ },+ { warn: vi.fn() } as never,+ metrics+ )++ const [superseded, notYet] = await resolver.resolveMany(principal, [+ event,+ {+ ...event,+ sourceEventId: 'outcome-ahead',+ sourceStatusRef: `host:mcp-host/chatllm-ahead:generation=7:uid=${HOST_UID}`,+ },+ ])++ expect(superseded).toEqual({ refusal: 'administrative_intent_generation_drift' })+ expect(notYet).toBeNull()+ // Exactly one of the two was counted: the metric names the terminal case+ // only, so a dashboard built on it is not inflated by the transient one.+ expect(metrics.inc).toHaveBeenCalledTimes(1)+ expect(metrics.inc).toHaveBeenCalledWith({ namespace: 'mcp-host' })+ })++ /**+ * Identity is checked before drift. A same-name Host with a different uid+ * is not the object the reporter observed, so its annotation says nothing+ * about this event — answering "terminal drift" there would turn a #694+ * recreation into a permanently dropped outcome.+ */+ it('does not report drift when the uid does not match', async () => {+ const metrics = { inc: vi.fn() }+ const other = host(undefined, 7, 6, '00000000-0000-4000-8000-000000000000')+ const listResource = vi.fn().mockResolvedValue([other])++ const result = await new HccAdministrativeOutcomeBindingResolver(+ { getResource: vi.fn(), listResource },+ { findHostIntent: vi.fn(), findHostIntents: vi.fn().mockResolvedValue(intentMap()) },+ { warn: vi.fn() } as never,+ metrics+ ).resolve(principal, event)++ expect(result).toBeNull()+ // Liveness witness for the negative assertion below: the resolver did+ // run the lookup and reach the candidate loop, so the uncounted metric+ // is a decision, not an unexecuted path.+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')+ expect(metrics.inc).not.toHaveBeenCalled()+ })+ })})diff --git a/control-api/test/services.resourceService.test.ts b/control-api/test/services.resourceService.test.tsindex 0223a70fb..5409111fb 100644--- a/control-api/test/services.resourceService.test.ts+++ b/control-api/test/services.resourceService.test.ts@@ -659,3 +659,183 @@ describe('ResourceService Host administrative intent', () => {expect(persistHostIntent).not.toHaveBeenCalled()})})++/**+ * #329 — the annotation must name the generation the apiserver PERSISTED, not+ * the one control-api predicted before the write. The Host CRD declares a+ * status subresource, so `metadata.generation` advances only on spec changes:+ * a replace whose spec is unchanged leaves the object at N while the+ * prediction writes N+1, and nothing ever retires that annotation, so the Host+ * stops binding outcomes permanently.+ */+describe('ResourceService administrative intent generation reconciliation (#329)', () => {+ const GENERATION = 'clerum.io/administrative-intent-generation'+ const OPERATION_ID = '11111111-1111-4111-8111-111111111111'++ function withIntent() {+ setAdministrativeOperationService({+ persistHostIntent: vi.fn().mockResolvedValue({+ operationId: OPERATION_ID,+ action: 'update',+ namespace: 'mcp-host',+ targetRef: 'mcp-host/host-a',+ operatorSub: 'admin-1',+ requestId: 'request-1',+ }),+ persistHostOutcome: vi.fn(),+ } as never)+ }++ /**+ * `replaced` is what the apiserver answers the replace with — the object as+ * persisted. `generation: 4` against a live 4 is the unchanged-spec case.+ */+ function api(replaced: Record<string, unknown>, patch = vi.fn().mockResolvedValue({})) {+ return {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion: '5',+ generation: 4,+ annotations: {},+ },+ spec: { contextRef: 'live' },+ }),+ replaceNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),+ patchNamespacedCustomObject: patch,+ listNamespacedCustomObject: vi.fn(),+ }+ }++ async function update(customApi: ReturnType<typeof api>) {+ withIntent()+ const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })+ try {+ return await runWithAdministrativeRequestContext(+ { operatorSub: 'admin-1', requestId: 'request-1' },+ () =>+ service.updateResource('hosts', 'host-a', { spec: { contextRef: 'next' } }, 'mcp-host')+ )+ } finally {+ setAdministrativeOperationService(null)+ }+ }++ it('lowers the annotation to the persisted generation when the replace did not bump it', async () => {+ const patch = vi.fn().mockResolvedValue({})+ const customApi = api(+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },+ patch+ )++ await update(customApi)++ // Liveness: the replace itself carried the PREDICTED value, so the patch+ // below is a correction of a real mismatch and not a no-op.+ const replaceBody = customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body+ expect(replaceBody.metadata.annotations[GENERATION]).toBe('5')++ expect(patch).toHaveBeenCalledOnce()+ const [patchArgs] = patch.mock.calls[0]!+ expect(patchArgs).toMatchObject({+ namespace: 'mcp-host',+ plural: 'hosts',+ name: 'host-a',+ body: {+ metadata: {+ // The precondition comes from the object the write returned, so a+ // concurrent change between the replace and the patch is refused+ // rather than overwritten.+ resourceVersion: '6',+ annotations: { [GENERATION]: '4' },+ },+ },+ })+ // Annotations only: a patch that carried a spec would bump the generation+ // it is correcting and feed itself.+ expect(Object.keys(patchArgs.body.metadata)).toEqual(['resourceVersion', 'annotations'])+ expect(patchArgs.body).not.toHaveProperty('spec')+ })++ it('does not patch when the apiserver assigned the predicted generation', async () => {+ const patch = vi.fn().mockResolvedValue({})+ const customApi = api(+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 5, resourceVersion: '6' } },+ patch+ )++ await update(customApi)++ // Liveness witness for the negative assertion: the write really happened+ // and really annotated 5, so the absent patch is the equality check and+ // not an unexecuted path.+ expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()+ expect(+ customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body.metadata.annotations[+ GENERATION+ ]+ ).toBe('5')+ expect(patch).not.toHaveBeenCalled()+ })++ /**+ * A 409 means the resourceVersion from the write is stale. The re-read+ * decides whether the correction is still ours to make: if the object has+ * moved past the generation THIS operation produced, writing either value+ * would attribute a generation the intent did not cause, so the method stops+ * and the binding resolver refuses those outcomes as drift — which is the+ * correct answer, not a regression.+ */+ it('stops correcting when a concurrent write moved the object past our generation', async () => {+ const conflict = makeConflictError()+ const patch = vi.fn().mockRejectedValue(conflict)+ const customApi = api(+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },+ patch+ )+ // The re-read sees a generation past the one our replace produced.+ customApi.getNamespacedCustomObject+ .mockResolvedValueOnce({+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion: '5',+ generation: 4,+ annotations: {},+ },+ spec: { contextRef: 'live' },+ })+ .mockResolvedValue({+ metadata: {+ name: 'host-a',+ namespace: 'mcp-host',+ resourceVersion: '9',+ generation: 6,+ annotations: {},+ },+ spec: { contextRef: 'other' },+ })++ await update(customApi)++ // Liveness: one patch WAS attempted and got the 409, so the single call is+ // the stop rule and not a method that never tried.+ expect(patch).toHaveBeenCalledOnce()+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)+ })++ it('reports rather than throws when the write response carries no generation', async () => {+ const patch = vi.fn().mockResolvedValue({})+ const customApi = api({ metadata: { name: 'host-a', namespace: 'mcp-host' } }, patch)++ // The mutation itself succeeded, so it must still return normally: turning+ // an uncorrectable annotation into a caller-visible error would report a+ // completed write as failed.+ await expect(update(customApi)).resolves.toMatchObject({+ metadata: { name: 'host-a' },+ })+ expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()+ expect(patch).not.toHaveBeenCalled()+ })+})diff --git a/control-api/test/services.routeTracingSubmission.test.ts b/control-api/test/services.routeTracingSubmission.test.tsindex f7524533f..a2984946e 100644--- a/control-api/test/services.routeTracingSubmission.test.ts+++ b/control-api/test/services.routeTracingSubmission.test.ts@@ -24,6 +24,7 @@ import type {TracingTransactionRunner,} from '../src/services/tracing/contracts.js'import {+ AdministrativeIntentGenerationDriftError,InvalidTracingInputError,RouteTracingSubmissionService,TracingBindingUnavailableError,@@ -196,6 +197,62 @@ describe('route tracing submission facade', () => {expect(appendManyInTransaction).not.toHaveBeenCalled()})+ /**+ * #329 — the classification trap. `rejectionReason` allowlists error classes+ * plus statuses [400, 403]; an unlisted 409 falls through to+ * `submission_failed`, which `operationalStatus.ts` treats as CRITICAL. A+ * drift refusal is an operator-attributable rejection, not a control-api+ * malfunction, so mislabelling it would trade a 403 log loop for a critical+ * alert per drifted Host.+ */+ it('classifies a generation-drift refusal as event_rejected, not submission_failed', async () => {+ const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')+ const operationalError = vi.spyOn(governedTraceOperationalErrorsTotal, 'inc')+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')+ const h = transactionHarness()+ const appendManyInTransaction = vi.fn()+ const service = new RouteTracingSubmissionService({+ transaction: h.transaction,+ administrativeOperationBindingResolver: {+ resolve: vi.fn(),+ resolveMany: vi+ .fn()+ .mockResolvedValue([{ refusal: 'administrative_intent_generation_drift' }]),+ },+ administrativeEventAppender: { appendManyInTransaction },+ })++ const failure = service.submit({+ principal: administrativePrincipal,+ events: [administrativeInput],+ })+ await expect(failure).rejects.toBeInstanceOf(AdministrativeIntentGenerationDriftError)+ await expect(failure).rejects.toMatchObject({+ code: 'administrative_intent_generation_drift',+ status: 409,+ statusCode: 409,+ eventIndex: 0,+ })++ // Liveness: the rejection path ran to completion — the event was counted+ // as rejected and the error clock was stamped.+ expect(rejected).toHaveBeenCalledOnce()+ expect(lastError).toHaveBeenCalledWith(+ { scope: 'administrative', reason: 'event_rejected' },+ expect.any(Number)+ )+ expect(lastError).not.toHaveBeenCalledWith(+ { scope: 'administrative', reason: 'submission_failed' },+ expect.any(Number)+ )+ expect(operationalError).not.toHaveBeenCalledWith({+ scope: 'administrative',+ reason: 'submission_failed',+ })+ expect(h.transactionSpy).not.toHaveBeenCalled()+ expect(appendManyInTransaction).not.toHaveBeenCalled()+ })+it('records accepted and replayed events with only bounded trace labels after commit', async () => {const accepted = vi.spyOn(governedTraceAcceptedTotal, 'inc')const replayed = vi.spyOn(governedTraceReplayedTotal, 'inc')diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex afd9bad63..a0a12dbe2 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.346",+ "version": "0.1.347","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.346",+ "version": "0.1.347","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 24992abae..2fbf1bd8d 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.346",+ "version": "0.1.347","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/administrativeOutcomeReporter.test.ts b/host-context-controller/src/administrativeOutcomeReporter.test.tsindex ccf89e93a..6b42c387f 100644--- a/host-context-controller/src/administrativeOutcomeReporter.test.ts+++ b/host-context-controller/src/administrativeOutcomeReporter.test.ts@@ -346,4 +346,50 @@ describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)expect(submittedIds(fetchFn)).toEqual(['first', 'second', 'third', 'first'])expect(await counted('deduplicated')).toBe(1)})++ /**+ * #329 — the loop this cuts. A Host whose intent annotation names a+ * generation it has already passed answers 409+ * `administrative_intent_generation_drift` forever, and before this change+ * that answer was a plain retryable failure: 3 submits per enqueue, and+ * `onDrop` deleted the dedupe key so the next reconciler pass enqueued it+ * again. Measured in dev at ~108 requests/hour for a single drifted Host.+ *+ * The terminal classification routes it to `onTerminal`, which PRESERVES the+ * key, so the re-enqueue is deduplicated instead of resubmitted. The bound+ * is honest: 1 submit, then 0 while the key stays resident — not zero, and+ * not permanent, since `seen` is memory and does not survive a restart.+ */+ it('settles a 409 generation-drift refusal once and stops re-sending it (#329)', async () => {+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {+ const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })+ .events[0]!.sourceEventId+ return id === 'drifted'+ ? failedResponse(409, {+ code: 'administrative_intent_generation_drift',+ error: 'ignored',+ })+ : failedResponse(500, { error: 'Internal server error' })+ }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('drifted'))+ reporter.enqueueHostOutcome(outcome('transient'))+ await settle()+ // The reconciler observes the same drifted Host on its next pass.+ reporter.enqueueHostOutcome(outcome('drifted'))+ await settle()++ const ids = submittedIds(fetchFn)+ // Liveness witness: a retryable failure on another key in the same run+ // used its full budget of 3, so the single 'drifted' submit is the+ // terminal classification and not a reporter that stopped submitting.+ expect(ids.filter(id => id === 'transient')).toHaveLength(3)+ expect(ids.filter(id => id === 'drifted')).toHaveLength(1)+ expect(await counted('rejected')).toBe(1)+ expect(await counted('deduplicated')).toBe(1)+ // Not a conflict: that label belongs to `tracing_idempotency_conflict`,+ // and conflating them would hide the drift inside an existing metric.+ expect(await counted('conflict')).toBe(0)+ })})diff --git a/host-context-controller/src/reporterHttpFailure.test.ts b/host-context-controller/src/reporterHttpFailure.test.tsindex e2af52fc7..a0a2030d5 100644--- a/host-context-controller/src/reporterHttpFailure.test.ts+++ b/host-context-controller/src/reporterHttpFailure.test.ts@@ -34,6 +34,7 @@ describe('throwForFailedSubmit', () => {it.each([[409, 'tracing_idempotency_conflict', 'conflict'],+ [409, 'administrative_intent_generation_drift', 'rejected'],[400, 'unsafe_tracing_input', 'rejected'],[400, 'invalid_tracing_input', 'rejected'],] as const)('treats %i %s as terminal %s', async (status, code, result) => {@@ -43,11 +44,33 @@ describe('throwForFailedSubmit', () => {expect(error).toMatchObject({ result })})+ /**+ * Two codes now share status 409 (#329), so the `find` on `code` decides+ * which terminal result applies rather than merely confirming the only+ * candidate. A resolver that returned the first 409 entry would answer+ * `conflict` for a drift refusal, and HCC would record a routine operator+ * outcome as an idempotency collision.+ */+ it('picks the terminal result by code when two entries share status 409', async () => {+ const conflict = await failure(409, async () => ({ code: 'tracing_idempotency_conflict' }))+ const drift = await failure(409, async () => ({+ code: 'administrative_intent_generation_drift',+ }))++ expect(conflict).toMatchObject({ result: 'conflict' })+ expect(drift).toMatchObject({ result: 'rejected' })+ })+it.each([[400, 'tracing_idempotency_conflict'],[409, 'unsafe_tracing_input'],[409, 'invalid_tracing_input'],[403, 'tracing_idempotency_conflict'],+ // The drift code carried on a 403 is NOT terminal. The 403 next door is+ // `tracing_binding_unavailable`, which also covers a binding that is+ // merely not visible yet; keeping the whole status retryable is what lets+ // a control-api/HCC deploy overlap heal itself (#329).+ [403, 'administrative_intent_generation_drift'],] as const)('keeps %i with %s retryable: the status is part of the pair',async (status, code) => {diff --git a/host-context-controller/src/reporterHttpFailure.ts b/host-context-controller/src/reporterHttpFailure.tsindex a295abd88..043517ba2 100644--- a/host-context-controller/src/reporterHttpFailure.ts+++ b/host-context-controller/src/reporterHttpFailure.ts@@ -12,6 +12,17 @@ const log = hccLogger.child({ module: 'reporter-http-failure' })* not read (#693) — and neither becomes acceptable on a retry, so it is as* final as `unsafe_tracing_input`. Every other failure (403 binding not yet* visible, 5xx, a 4xx without a code, a body that is not JSON) stays retryable.+ *+ * `administrative_intent_generation_drift` (#329) is deterministic for a+ * different reason than the others: the request is well formed and the server+ * state is real, but the Host carries an intent annotation pinned to a+ * generation it has already passed, and nothing ever retires that annotation.+ * Generations only grow and the reporter always reads the live object, so the+ * answer cannot change. The plain 403 next door is NOT deterministic — it also+ * covers a binding that is merely not visible yet, which is what lets a+ * control-api/HCC deploy overlap heal itself — so only the coded 409 is listed+ * here. Two entries now share status 409, which makes the `find` on `code`+ * below load-bearing rather than incidental.*/const TERMINAL_RESPONSES: ReadonlyArray<{status: number@@ -19,6 +30,7 @@ const TERMINAL_RESPONSES: ReadonlyArray<{result: ReporterTerminalResult}> = [{ status: 409, code: 'tracing_idempotency_conflict', result: 'conflict' },+ { status: 409, code: 'administrative_intent_generation_drift', result: 'rejected' },{ status: 400, code: 'unsafe_tracing_input', result: 'rejected' },{ status: 400, code: 'invalid_tracing_input', result: 'rejected' },]