Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(tracing): retire drifted administrative intent annotations (#329)

Alfredo Lopeza90b619b61e1
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 198596027..6ae781fe2 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.593",
+ "version": "0.1.594",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.593",
+ "version": "0.1.594",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 68e2a3c02..5b35eb47d 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.593",
+ "version": "0.1.594",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/http/errorHandler.ts b/control-api/src/http/errorHandler.ts
index 25f61b94a..876f2f263 100644
--- a/control-api/src/http/errorHandler.ts
+++ b/control-api/src/http/errorHandler.ts
@@ -20,6 +20,11 @@ export const FORWARDABLE_CLIENT_ERROR_CODES = new Set([
'tracing_idempotency_conflict',
'unsafe_tracing_input',
'invalid_tracing_input',
+ // Deterministic: the annotated generation is already superseded, so no
+ // future report can bind. `tracing_binding_unavailable` stays OUT on
+ // purpose — that 403 must remain code-less and therefore retryable, which
+ // is what lets a control-api/HCC deploy overlap heal itself.
+ 'administrative_intent_generation_drift',
])
/**
diff --git a/control-api/src/observability/metrics.ts b/control-api/src/observability/metrics.ts
index 4ad66acfa..170ba8483 100644
--- a/control-api/src/observability/metrics.ts
+++ b/control-api/src/observability/metrics.ts
@@ -406,6 +406,22 @@ export const governedTraceConflictingTotal = getOrCreateCounter({
labelNames: ['family', 'source', 'type'] as const as Array<'family' | 'source' | 'type'>,
})
+/**
+ * A Host whose administrative-intent annotation names a generation the object
+ * has already passed, so no outcome for it can ever be attributed (#329). The
+ * authoritative drift signal: HCC's own reporter counter labels this
+ * `result="rejected"`, the same label it uses for malformed input, and cannot
+ * tell the two apart.
+ *
+ * `namespace` is the only label. Host names are unbounded, so putting one in a
+ * label would make the series count grow with the cluster.
+ */
+export const governedTraceAdministrativeIntentDriftTotal = getOrCreateCounter({
+ name: 'governed_trace_administrative_intent_drift_total',
+ help: 'Count of administrative tracing events refused because the intent annotation names a superseded generation.',
+ labelNames: ['namespace'] as const as Array<'namespace'>,
+})
+
export const governedTraceIngestDurationSeconds = getOrCreateHistogram({
name: 'governed_trace_ingest_duration_seconds',
help: 'Duration of governed trace ingestion and stream registration.',
diff --git a/control-api/src/services/resourceService.ts b/control-api/src/services/resourceService.ts
index ccb26a7ef..ca25ca2cb 100644
--- a/control-api/src/services/resourceService.ts
+++ b/control-api/src/services/resourceService.ts
@@ -1,4 +1,5 @@
import * as k8s from '@kubernetes/client-node'
+import { rootLogger } from '../observability/logger.js'
import {
CLERUM_GROUP,
CLERUM_VERSION,
@@ -12,6 +13,7 @@ import {
kindFromPlural,
parseProjectedGeneration,
} from './resourceServiceHelpers.js'
+import { ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION } from './tracing/adminOperationConstants.js'
import { currentAdministrativeRequestContext } from './tracing/adminOperationContext.js'
import {
ControlApiAdministrativeOperationService,
@@ -253,8 +255,16 @@ export class ResourceService {
name: body.metadata.name,
})
const sanitizedMetadata = stripAdministrativeIntentAnnotation(body.metadata) ?? body.metadata
+ // A create is the one prediction that is almost always right — the
+ // apiserver assigns generation 1 — but it is still a prediction, and an
+ // admission webhook that mutates the spec can land the object at 2.
+ const predictedGeneration = 1
const annotations = intent
- ? withAdministrativeIntentAnnotation(sanitizedMetadata.annotations, intent, 1)
+ ? withAdministrativeIntentAnnotation(
+ sanitizedMetadata.annotations,
+ intent,
+ predictedGeneration
+ )
: sanitizedMetadata.annotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
@@ -268,8 +278,9 @@ export class ResourceService {
spec: body.spec,
}
+ let created: unknown
try {
- return await this.customApi.createNamespacedCustomObject({
+ created = await this.customApi.createNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
@@ -280,6 +291,18 @@ export class ResourceService {
await persistHostFailure(intent)
throw err
}
+ // Outside the try on purpose: the create already succeeded, and a failure
+ // to correct the annotation must not reach `persistHostFailure`, which
+ // would record a completed operation as failed.
+ await this.reconcileAdministrativeIntentGeneration({
+ plural,
+ name: body.metadata.name,
+ namespace: ns,
+ intent,
+ predictedGeneration,
+ written: created,
+ })
+ return created
}
async updateResource(
@@ -343,12 +366,15 @@ export class ResourceService {
current.metadata?.annotations,
sanitizedMetadata?.annotations
)
+ // Predicted, not persisted: the annotation travels inside the same body
+ // as the spec, so the generation the apiserver will assign is not
+ // knowable yet. `reconcileAdministrativeIntentGeneration` corrects it
+ // against the response — a replace whose spec is unchanged leaves the
+ // generation where it was, and the prediction would pin the intent to a
+ // generation that never arrives.
+ const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
const annotations = intent
- ? withAdministrativeIntentAnnotation(
- mergedAnnotations,
- intent,
- Math.max(1, (current.metadata?.generation ?? 0) + 1)
- )
+ ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
: mergedAnnotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
@@ -363,8 +389,9 @@ export class ResourceService {
spec: body.spec,
}
+ let replaced: unknown
try {
- return await this.customApi.replaceNamespacedCustomObject({
+ replaced = await this.customApi.replaceNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
@@ -393,10 +420,177 @@ export class ResourceService {
await persistHostFailure(intent)
throw err
}
+ // After the catch, not inside the try: the replace already succeeded, so
+ // a failure here must not reach `persistHostFailure` and record a
+ // completed operation as failed.
+ await this.reconcileAdministrativeIntentGeneration({
+ plural,
+ name,
+ namespace: ns,
+ intent,
+ predictedGeneration,
+ written: replaced,
+ })
+ return replaced
}
throw new Error(`Failed to update ${plural}/${name} after ${maxAttempts} attempts`)
}
+ /**
+ * Replace the PREDICTED generation in the administrative-intent annotation
+ * with the one the apiserver actually persisted (#329).
+ *
+ * The three write paths above annotate with `current.generation + 1` before
+ * the write, because the annotation has to travel inside the same body. That
+ * prediction is wrong whenever the write does not bump the generation, and
+ * the Host CRD declares a status subresource
+ * (`charts/clerum-crds/crds/host.yaml`), so `metadata.generation` advances
+ * ONLY on spec changes. A replace whose spec is byte-identical — an
+ * idempotent operator PUT, a `mutateResource` whose mutation is a no-op, a
+ * hook-ref resync — leaves the object at N while the annotation says N+1.
+ * Nothing ever retires that annotation, so the Host stops binding outcomes
+ * forever.
+ *
+ * A merge-patch of annotations alone does not bump the generation, so this
+ * correction cannot feed itself.
+ *
+ * NOT `patchAnnotationMonotonic`: that method takes the max of projected and
+ * requested and refuses to lower a value. Lowering N+1 back to N is exactly
+ * the correction needed here.
+ *
+ * Never throws. The write it corrects already succeeded, and turning a
+ * successful mutation into a failed request would misreport what happened to
+ * the object. Everything it cannot do is logged with the reason.
+ */
+ private async reconcileAdministrativeIntentGeneration(input: {
+ plural: ClerumResourceType
+ name: string
+ namespace: string
+ intent: HostAdministrativeIntent | null
+ predictedGeneration: number
+ written: unknown
+ }): Promise<void> {
+ if (!input.intent) return
+ const metadata = (input.written as { metadata?: { generation?: number } } | null)?.metadata
+ const persisted = metadata?.generation
+ if (typeof persisted !== 'number' || !Number.isSafeInteger(persisted)) {
+ rootLogger.warn(
+ {
+ event: 'administrative_intent_generation_unverified',
+ plural: input.plural,
+ namespace: input.namespace,
+ name: input.name,
+ predictedGeneration: input.predictedGeneration,
+ },
+ 'write response carried no usable metadata.generation; the intent annotation was left at the predicted value'
+ )
+ return
+ }
+ if (persisted === input.predictedGeneration) return
+
+ const maxAttempts = 3
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+ // The first attempt uses the write's own response; a retry re-reads,
+ // because a 409 means the resourceVersion we held is stale.
+ let snapshot: { metadata?: { generation?: number; resourceVersion?: string } }
+ if (attempt === 1) {
+ snapshot = input.written as { metadata?: { generation?: number; resourceVersion?: string } }
+ } else {
+ try {
+ snapshot = (await this.getResource(input.plural, input.name, input.namespace)) as {
+ metadata?: { generation?: number; resourceVersion?: string }
+ }
+ } catch (err) {
+ // The only place in this method that can throw. Caught so the
+ // documented contract holds, and reported rather than swallowed.
+ rootLogger.warn(
+ {
+ event: 'administrative_intent_generation_patch_failed',
+ plural: input.plural,
+ namespace: input.namespace,
+ name: input.name,
+ predictedGeneration: input.predictedGeneration,
+ persistedGeneration: persisted,
+ status: extractK8sStatus(err),
+ },
+ 're-read failed while correcting the intent annotation to the persisted generation'
+ )
+ return
+ }
+ }
+ if (snapshot.metadata?.generation !== persisted) {
+ // Something else advanced the object past the generation this
+ // operation produced. Writing `persisted` now would attribute a
+ // generation this intent did not cause; writing the newer one would
+ // do the same. The intent is genuinely unattributable from here, and
+ // the binding resolver refuses its outcomes by design (#329).
+ rootLogger.warn(
+ {
+ event: 'administrative_intent_generation_superseded',
+ plural: input.plural,
+ namespace: input.namespace,
+ name: input.name,
+ writtenGeneration: persisted,
+ liveGeneration: snapshot.metadata?.generation,
+ },
+ 'object advanced past the generation this operation produced; intent annotation left uncorrected'
+ )
+ return
+ }
+ const resourceVersion = snapshot.metadata?.resourceVersion
+ if (!resourceVersion) {
+ rootLogger.warn(
+ {
+ event: 'administrative_intent_generation_unverified',
+ plural: input.plural,
+ namespace: input.namespace,
+ name: input.name,
+ predictedGeneration: input.predictedGeneration,
+ persistedGeneration: persisted,
+ },
+ 'no metadata.resourceVersion to use as a patch precondition; the intent annotation was left at the predicted value'
+ )
+ return
+ }
+ try {
+ await this.customApi.patchNamespacedCustomObject(
+ {
+ group: CLERUM_GROUP,
+ version: CLERUM_VERSION,
+ namespace: input.namespace,
+ plural: input.plural,
+ name: input.name,
+ body: {
+ metadata: {
+ resourceVersion,
+ annotations: {
+ [ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION]: String(persisted),
+ },
+ },
+ },
+ },
+ k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch)
+ )
+ return
+ } catch (err) {
+ if (extractK8sStatus(err) === 409 && attempt < maxAttempts) continue
+ rootLogger.warn(
+ {
+ event: 'administrative_intent_generation_patch_failed',
+ plural: input.plural,
+ namespace: input.namespace,
+ name: input.name,
+ predictedGeneration: input.predictedGeneration,
+ persistedGeneration: persisted,
+ status: extractK8sStatus(err),
+ },
+ 'could not correct the intent annotation to the persisted generation'
+ )
+ return
+ }
+ }
+ }
+
/**
* Merge-patch ONLY metadata.annotations of a custom object.
*
@@ -563,12 +757,15 @@ export class ResourceService {
current.metadata?.annotations,
sanitizedMetadata?.annotations
)
+ // Predicted, not persisted. This path is the likeliest producer of drift
+ // in the repository: it does not short-circuit on an unchanged spec (the
+ // `if (!next)` above only fires when the mutation returns nothing), so a
+ // mutation that computes the same spec still issues a replace, and the
+ // Host CRD's status subresource means such a replace leaves the
+ // generation untouched while the prediction moves ahead of it.
+ const predictedGeneration = Math.max(1, (current.metadata?.generation ?? 0) + 1)
const annotations = intent
- ? withAdministrativeIntentAnnotation(
- mergedAnnotations,
- intent,
- Math.max(1, (current.metadata?.generation ?? 0) + 1)
- )
+ ? withAdministrativeIntentAnnotation(mergedAnnotations, intent, predictedGeneration)
: mergedAnnotations
const resource: ClerumResource = {
apiVersion: `${CLERUM_GROUP}/${CLERUM_VERSION}`,
@@ -583,8 +780,9 @@ export class ResourceService {
spec: next.spec,
}
+ let replaced: unknown
try {
- return await this.customApi.replaceNamespacedCustomObject({
+ replaced = await this.customApi.replaceNamespacedCustomObject({
group: CLERUM_GROUP,
version: CLERUM_VERSION,
namespace: ns,
@@ -605,6 +803,15 @@ export class ResourceService {
await persistHostFailure(intent)
throw err
}
+ await this.reconcileAdministrativeIntentGeneration({
+ plural,
+ name,
+ namespace: ns,
+ intent,
+ predictedGeneration,
+ written: replaced,
+ })
+ return replaced
}
throw new Error(`Failed to mutate ${plural}/${name} after ${maxAttempts} attempts`)
}
diff --git a/control-api/src/services/tracing/adminOperationBindingResolver.ts b/control-api/src/services/tracing/adminOperationBindingResolver.ts
index 026a9c4ee..9bd75c6ec 100644
--- a/control-api/src/services/tracing/adminOperationBindingResolver.ts
+++ b/control-api/src/services/tracing/adminOperationBindingResolver.ts
@@ -1,4 +1,6 @@
import type { AdministrativeEventSubmitterPrincipalV1 } from '../../middleware/tracingSubmitterAuth.js'
+import { type Logger, rootLogger } from '../../observability/logger.js'
+import { governedTraceAdministrativeIntentDriftTotal } from '../../observability/metrics.js'
import {
ADMINISTRATIVE_INTENT_ANNOTATION,
ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION,
@@ -8,7 +10,11 @@ import {
administrativeIntentLookupKey,
} from './adminOperationService.js'
import type { AdministrativeEventInputV1, AdministrativeServerBindingV1 } from './contracts.js'
-import type { AdministrativeOperationBindingResolver } from './routeSubmissionService.js'
+import {
+ type AdministrativeOperationBindingResolver,
+ type BindingRefusal,
+ isBindingRefusal,
+} from './routeSubmissionService.js'
export interface HostAdministrativeLookup {
getResource(plural: 'hosts', name: string, namespace: string): Promise<unknown>
@@ -45,23 +51,43 @@ const STATUS_REF =
/^host:([a-z0-9]([-a-z0-9]*[a-z0-9])?)\/([a-z0-9]([-a-z0-9]*[a-z0-9])?):generation=([1-9][0-9]*):uid=([0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+const DRIFT_REFUSAL: BindingRefusal = { refusal: 'administrative_intent_generation_drift' }
+
+type Candidate = {
+ reference: { namespace: string; name: string; generation: number; uid: string }
+ operationId: string
+ targetRef: string
+ outcome: 'succeeded' | 'failed'
+}
+
+/** The one counter method this resolver uses, so a test can pass a fake. */
+export type DriftCounter = { inc(labels: { namespace: string }): void }
+
export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOperationBindingResolver {
+ private readonly logger: Logger
+ private readonly metrics: DriftCounter
+
constructor(
private readonly hostLookup: HostAdministrativeLookup,
- private readonly intentLookup: AdministrativeIntentLookup
- ) {}
+ private readonly intentLookup: AdministrativeIntentLookup,
+ logger: Logger = rootLogger,
+ metrics: DriftCounter = governedTraceAdministrativeIntentDriftTotal
+ ) {
+ this.logger = logger
+ this.metrics = metrics
+ }
async resolve(
principal: AdministrativeEventSubmitterPrincipalV1,
event: AdministrativeEventInputV1
- ): Promise<AdministrativeServerBindingV1 | null> {
+ ): Promise<AdministrativeServerBindingV1 | BindingRefusal | null> {
return (await this.resolveMany(principal, [event]))[0] ?? null
}
async resolveMany(
principal: AdministrativeEventSubmitterPrincipalV1,
events: readonly AdministrativeEventInputV1[]
- ): Promise<readonly (AdministrativeServerBindingV1 | null)[]> {
+ ): Promise<readonly (AdministrativeServerBindingV1 | BindingRefusal | null)[]> {
if (principal.kind !== 'hcc_internal_control') {
return events.map(() => null)
}
@@ -105,25 +131,49 @@ export class HccAdministrativeOutcomeBindingResolver implements AdministrativeOp
metadata.annotations?.[ADMINISTRATIVE_INTENT_GENERATION_ANNOTATION]
)
const targetRef = `${reference.namespace}/${reference.name}`
- if (
- !operationId ||
- !UUID.test(operationId) ||
- !Number.isSafeInteger(expectedGeneration) ||
- expectedGeneration !== reference.generation
- )
+ if (!operationId || !UUID.test(operationId) || !Number.isSafeInteger(expectedGeneration))
return null
+ if (expectedGeneration < reference.generation) {
+ // Every identity field above already matched, so the live object IS the
+ // one the reporter observed, and its generation has moved past the one
+ // the annotation pins. Generations only grow and nothing ever retires
+ // the annotation, so this pair can never bind again: terminal, not
+ // "unavailable right now".
+ this.metrics.inc({ namespace: reference.namespace })
+ // The Host name goes to the log, never to the metric label: names are
+ // unbounded and a label carrying one grows the series count with the
+ // cluster. The operationId is omitted from both.
+ this.logger.warn(
+ {
+ event: 'administrative_intent_generation_drift',
+ namespace: reference.namespace,
+ name: reference.name,
+ liveGeneration: reference.generation,
+ annotatedGeneration: expectedGeneration,
+ },
+ 'administrative intent annotation is pinned to a superseded generation'
+ )
+ return DRIFT_REFUSAL
+ }
+ // `expectedGeneration > reference.generation` stays retryable: that is the
+ // window between control-api predicting a generation and the corrective
+ // annotation patch landing, and it resolves on its own.
+ if (expectedGeneration !== reference.generation) return null
const outcome = safeOutcome(events[index]!.payload?.status)
if (!outcome) return null
return { reference, operationId, targetRef, outcome }
})
- const intentInputs = candidates.filter(Boolean).map(candidate => ({
- operationId: candidate!.operationId,
- targetRef: candidate!.targetRef,
- namespace: candidate!.reference.namespace,
- }))
+ const intentInputs = candidates
+ .filter(candidate => candidate !== null && !isBindingRefusal(candidate))
+ .map(candidate => ({
+ operationId: (candidate as Candidate).operationId,
+ targetRef: (candidate as Candidate).targetRef,
+ namespace: (candidate as Candidate).reference.namespace,
+ }))
const intents = await this.intentLookup.findHostIntents(intentInputs)
return candidates.map(candidate => {
if (!candidate) return null
+ if (isBindingRefusal(candidate)) return candidate
const intentInput = {
operationId: candidate.operationId,
targetRef: candidate.targetRef,
diff --git a/control-api/src/services/tracing/routeSubmissionService.ts b/control-api/src/services/tracing/routeSubmissionService.ts
index 1049a5bc0..f3ecab482 100644
--- a/control-api/src/services/tracing/routeSubmissionService.ts
+++ b/control-api/src/services/tracing/routeSubmissionService.ts
@@ -62,15 +62,37 @@ export interface AgentRunBindingResolver {
): MaybePromise<readonly (AgentRunServerBindingV1 | null | undefined)[]>
}
+/**
+ * A resolver's way of saying "this event can never bind", as opposed to `null`,
+ * which means "it cannot bind right now". The two need different HTTP statuses
+ * because HCC retries one and gives up on the other.
+ *
+ * It is a value rather than a throw so the resolver can answer per event while
+ * `resolveTrustedBindings` keeps the event's index — the same reason `null` is
+ * a value there. It does NOT mean the batch continues: the first unbindable
+ * event still aborts the whole submission (see `resolveTrustedBindings`).
+ */
+export type BindingRefusal = {
+ readonly refusal: 'administrative_intent_generation_drift'
+}
+
+export function isBindingRefusal(value: unknown): value is BindingRefusal {
+ return (
+ typeof value === 'object' &&
+ value !== null &&
+ (value as { refusal?: unknown }).refusal === 'administrative_intent_generation_drift'
+ )
+}
+
export interface AdministrativeOperationBindingResolver {
resolve(
principal: AdministrativeEventSubmitterPrincipalV1,
event: AdministrativeEventInputV1
- ): MaybePromise<AdministrativeServerBindingV1 | null | undefined>
+ ): MaybePromise<AdministrativeServerBindingV1 | BindingRefusal | null | undefined>
resolveMany?(
principal: AdministrativeEventSubmitterPrincipalV1,
events: readonly AdministrativeEventInputV1[]
- ): MaybePromise<readonly (AdministrativeServerBindingV1 | null | undefined)[]>
+ ): MaybePromise<readonly (AdministrativeServerBindingV1 | BindingRefusal | null | undefined)[]>
}
export interface InfrastructureWorkloadBindingResolver {
@@ -85,11 +107,14 @@ export interface InfrastructureWorkloadBindingResolver {
}
type BindingResolver<Principal, Event, Binding> = {
- resolve(principal: Principal, event: Event): MaybePromise<Binding | null | undefined>
+ resolve(
+ principal: Principal,
+ event: Event
+ ): MaybePromise<Binding | BindingRefusal | null | undefined>
resolveMany?(
principal: Principal,
events: readonly Event[]
- ): MaybePromise<readonly (Binding | null | undefined)[]>
+ ): MaybePromise<readonly (Binding | BindingRefusal | null | undefined)[]>
}
export class TracingBindingUnavailableError extends Error {
@@ -106,6 +131,30 @@ export class TracingBindingUnavailableError extends Error {
}
}
+/**
+ * The reported object carries an administrative-intent annotation pinned to a
+ * generation the object has already passed, so no future report can ever bind:
+ * the reporter always reads the live object and `metadata.generation` only
+ * grows. Deterministic by construction, which is why it is a 409 with a
+ * forwarded code and not the retryable 403 that `TracingBindingUnavailableError`
+ * produces.
+ *
+ * The message carries no Host identity: `name`/`namespace`/`uid` go to the
+ * structured log, never to a response body.
+ */
+export class AdministrativeIntentGenerationDriftError extends Error {
+ readonly code = 'administrative_intent_generation_drift'
+ readonly status = 409
+ readonly statusCode = 409
+
+ constructor(readonly eventIndex: number) {
+ super(
+ `administrative intent annotation is pinned to a superseded generation for tracing event ${eventIndex}`
+ )
+ this.name = 'AdministrativeIntentGenerationDriftError'
+ }
+}
+
export class InvalidTracingInputError extends TypeError {
readonly code = 'invalid_tracing_input'
readonly status = 400
@@ -375,7 +424,12 @@ async function resolveTrustedBindings<Principal, Event, Binding>(input: {
`trusted ${input.bindingKind} binding resolver returned an unexpected batch size`
)
}
+ // All-or-nothing, unchanged: the first event that does not resolve throws
+ // before any append runs, so a batch holding one drifted event is refused
+ // whole and nothing is stored. A refusal only picks a different status than
+ // `null` does; it does not let the batch continue.
return resolved.map((binding, index) => {
+ if (isBindingRefusal(binding)) throw new AdministrativeIntentGenerationDriftError(index)
if (!binding) throw new TracingBindingUnavailableError(input.bindingKind, index)
return binding
})
@@ -444,6 +498,11 @@ function rejectionReason(
error instanceof InvalidTracingInputError ||
error instanceof UnsafeTracingInputError ||
error instanceof TracingBindingUnavailableError ||
+ // A 409 that is not an idempotency conflict falls past the status check
+ // below (which only lists 400 and 403) into `submission_failed`, which
+ // operationalStatus treats as critical. Naming it here keeps a drifted
+ // Host an event-level rejection instead of a critical service alert.
+ error instanceof AdministrativeIntentGenerationDriftError ||
(error !== null &&
typeof error === 'object' &&
('status' in error || 'statusCode' in error) &&
diff --git a/control-api/test/http.errorHandler.clientErrorCode.test.ts b/control-api/test/http.errorHandler.clientErrorCode.test.ts
index 09731b17d..2db8129b3 100644
--- a/control-api/test/http.errorHandler.clientErrorCode.test.ts
+++ b/control-api/test/http.errorHandler.clientErrorCode.test.ts
@@ -8,6 +8,7 @@ import {
} from '../src/services/tracing/append.js'
import { GovernedReadInvalidQueryError } from '../src/services/tracing/governedEventReadService.js'
import {
+ AdministrativeIntentGenerationDriftError,
InvalidTracingInputError,
TracingBindingUnavailableError,
} from '../src/services/tracing/routeSubmissionService.js'
@@ -33,6 +34,11 @@ describe('clerumErrorHandler — machine-readable code on allowlisted 4xx', () =
],
[new UnsafeTracingInputError('payload.gfs_subject'), 400, 'unsafe_tracing_input'],
[new InvalidTracingInputError('events[0].kind is not supported'), 400, 'invalid_tracing_input'],
+ [
+ new AdministrativeIntentGenerationDriftError(0),
+ 409,
+ 'administrative_intent_generation_drift',
+ ],
])('forwards %s with its code', async (err, status, code) => {
const res = await request(appThrowing(err)).post('/boom')
diff --git a/control-api/test/routes.tracingSubmissionBoundary.test.ts b/control-api/test/routes.tracingSubmissionBoundary.test.ts
index 56d378dc9..7f37bc2f0 100644
--- a/control-api/test/routes.tracingSubmissionBoundary.test.ts
+++ b/control-api/test/routes.tracingSubmissionBoundary.test.ts
@@ -464,49 +464,61 @@ describe('internal tracing submission routers — rejection code on the wire', (
* proves the handler forwards the error it is handed; this one has to reach
* the real resolver through the real route.
*/
- it('answers an administrative outcome with a pre-uid sourceStatusRef 403 through the real route', async () => {
- const OPERATION_ID = '11111111-1111-4111-8111-111111111111'
- const listResource = vi.fn().mockResolvedValue([
- {
- apiVersion: 'clerum.io/v1alpha1',
- kind: 'Host',
- metadata: {
- name: 'chatllm',
- namespace: 'mcp-host',
- generation: 7,
- uid: HOST_UID,
- annotations: {
- 'clerum.io/administrative-intent-id': OPERATION_ID,
- 'clerum.io/administrative-intent-generation': '7',
- },
+ const OPERATION_ID = '11111111-1111-4111-8111-111111111111'
+
+ function hostResource(name: string, generation: number, intentGeneration: number) {
+ return {
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'Host',
+ metadata: {
+ name,
+ namespace: 'mcp-host',
+ generation,
+ uid: HOST_UID,
+ annotations: {
+ 'clerum.io/administrative-intent-id': OPERATION_ID,
+ 'clerum.io/administrative-intent-generation': String(intentGeneration),
},
},
- ])
- const findHostIntents = vi.fn().mockResolvedValue(
- new Map([
- [
- administrativeIntentLookupKey({
- operationId: OPERATION_ID,
- targetRef: 'mcp-host/chatllm',
- namespace: 'mcp-host',
- }),
- {
- operatorSub: 'admin-1',
- requestId: 'request-1',
- environment: 'test',
- tenantId: null,
- teamId: null,
- identityIssuer: 'control-api',
- operatorUserId: '22222222-2222-4222-8222-222222222222',
- resourceAud: 'control-ui',
- effectiveScopes: [],
- tokenExchangeId: null,
- authorizationDecision: 'allow' as const,
- decisionActorSub: 'control-api',
- },
- ],
+ }
+ }
+
+ function intentFor(...names: readonly string[]) {
+ return new Map(
+ names.map(name => [
+ administrativeIntentLookupKey({
+ operationId: OPERATION_ID,
+ targetRef: `mcp-host/${name}`,
+ namespace: 'mcp-host',
+ }),
+ {
+ operatorSub: 'admin-1',
+ requestId: 'request-1',
+ environment: 'test',
+ tenantId: null,
+ teamId: null,
+ identityIssuer: 'control-api',
+ operatorUserId: '22222222-2222-4222-8222-222222222222',
+ resourceAud: 'control-ui',
+ effectiveScopes: [],
+ tokenExchangeId: null,
+ authorizationDecision: 'allow' as const,
+ decisionActorSub: 'control-api',
+ },
])
)
+ }
+
+ /**
+ * The real administrative router over the real submission service, the real
+ * HCC resolver and the global handler. Only the API-server list and the
+ * database append are doubles.
+ */
+ function realAdministrativeApp(hosts: ReadonlyArray<ReturnType<typeof hostResource>>) {
+ const listResource = vi.fn().mockResolvedValue([...hosts])
+ const findHostIntents = vi
+ .fn()
+ .mockResolvedValue(intentFor(...hosts.map(host => host.metadata.name)))
const appendManyInTransaction = vi.fn().mockResolvedValue([
{
kind: 'accepted' as const,
@@ -535,23 +547,28 @@ describe('internal tracing submission routers — rejection code on the wire', (
)
)
app.use(clerumErrorHandler)
+ return { app, listResource, appendManyInTransaction }
+ }
- const post = (sourceStatusRef: string) =>
- request(app)
- .post('/internal/tracing/administrative-events')
- .set('Authorization', `Bearer ${signInternalControl('hcc')}`)
- .send({
- events: [
- {
- kind: 'linked_outcome',
- sourceEventId: `outcome-${sourceStatusRef.length}`,
- occurredAt: '2026-07-10T09:59:59.000Z',
- reasonCode: 'boundary_test',
- sourceStatusRef,
- payload: { resource_class: 'Host', status: 'succeeded' },
- },
- ],
- })
+ function postOutcomes(app: express.Express, refs: readonly string[]) {
+ return request(app)
+ .post('/internal/tracing/administrative-events')
+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)
+ .send({
+ events: refs.map((sourceStatusRef, index) => ({
+ kind: 'linked_outcome',
+ sourceEventId: `outcome-${index}-${sourceStatusRef.length}`,
+ occurredAt: '2026-07-10T09:59:59.000Z',
+ reasonCode: 'boundary_test',
+ sourceStatusRef,
+ payload: { resource_class: 'Host', status: 'succeeded' },
+ })),
+ })
+ }
+
+ it('answers an administrative outcome with a pre-uid sourceStatusRef 403 through the real route', async () => {
+ const { app, appendManyInTransaction } = realAdministrativeApp([hostResource('chatllm', 7, 7)])
+ const post = (sourceStatusRef: string) => postOutcomes(app, [sourceStatusRef])
// Liveness: the current format binds and stores through this same app, so
// the refusal below is the legacy format and not a route that refuses all.
@@ -586,4 +603,78 @@ describe('internal tracing submission routers — rejection code on the wire', (
expect(response.body.correlationId).toEqual(expect.any(String))
expect(response.body).not.toHaveProperty('code')
})
+
+ /**
+ * #329 through the real route. The unit test proves the resolver returns a
+ * refusal; only this proves the refusal survives the submission service, the
+ * router and the global handler as a 409 carrying its code — which is the
+ * only form HCC can classify as terminal.
+ */
+ describe('administrative intent generation drift (#329)', () => {
+ const LIVE_REF = `host:mcp-host/chatllm:generation=7:uid=${HOST_UID}`
+ const DRIFTED_REF = `host:mcp-host/drifted:generation=7:uid=${HOST_UID}`
+
+ it('answers a superseded intent annotation 409 with its code through the real route', async () => {
+ const { app, appendManyInTransaction } = realAdministrativeApp([
+ hostResource('chatllm', 7, 7),
+ hostResource('drifted', 7, 6),
+ ])
+
+ // Liveness: the healthy Host binds and stores through this same app, so
+ // the 409 below is the drifted annotation and not a route that refuses
+ // everything.
+ const accepted = await postOutcomes(app, [LIVE_REF])
+ expect(accepted.status).toBe(200)
+ expect(appendManyInTransaction).toHaveBeenCalledOnce()
+
+ const refused = await postOutcomes(app, [DRIFTED_REF])
+
+ expect(refused.status).toBe(409)
+ expect(refused.body).toMatchObject({
+ code: 'administrative_intent_generation_drift',
+ correlationId: expect.any(String),
+ })
+ // Nothing was appended for the refused request: the count is still the
+ // one call the liveness probe made.
+ expect(appendManyInTransaction).toHaveBeenCalledOnce()
+ })
+
+ it('keeps Host identity out of the refusal body', async () => {
+ const { app } = realAdministrativeApp([hostResource('drifted', 7, 6)])
+
+ const refused = await postOutcomes(app, [DRIFTED_REF])
+
+ expect(refused.status).toBe(409)
+ // Liveness: the refusal really is the drift branch, so the absence of
+ // the name below is not an unrelated failure with an empty body.
+ expect(refused.body.code).toBe('administrative_intent_generation_drift')
+ const serialized = JSON.stringify(refused.body)
+ expect(serialized).not.toContain('drifted')
+ expect(serialized).not.toContain('mcp-host')
+ expect(serialized).not.toContain(HOST_UID)
+ })
+
+ /**
+ * `resolveTrustedBindings` throws on the first event it cannot resolve,
+ * before any append runs, so a batch holding one drifted event is refused
+ * whole. The refusal changes which status the batch gets; it does not let
+ * the batch continue past the bad event.
+ */
+ it('refuses a batch whole when one of its events has drifted', async () => {
+ const { app, listResource, appendManyInTransaction } = realAdministrativeApp([
+ hostResource('chatllm', 7, 7),
+ hostResource('drifted', 7, 6),
+ ])
+
+ const mixed = await postOutcomes(app, [LIVE_REF, DRIFTED_REF])
+
+ expect(mixed.status).toBe(409)
+ expect(mixed.body.code).toBe('administrative_intent_generation_drift')
+ // Liveness: the resolver did run over the whole batch — it read the
+ // Hosts — so "nothing stored" is the all-or-nothing rule and not a
+ // request that never reached the service.
+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')
+ expect(appendManyInTransaction).not.toHaveBeenCalled()
+ })
+ })
})
diff --git a/control-api/test/services.adminOperationBindingResolver.test.ts b/control-api/test/services.adminOperationBindingResolver.test.ts
index dc44669bb..5c68563ac 100644
--- a/control-api/test/services.adminOperationBindingResolver.test.ts
+++ b/control-api/test/services.adminOperationBindingResolver.test.ts
@@ -124,7 +124,10 @@ describe('HccAdministrativeOutcomeBindingResolver', () => {
it.each([
['stale generation', host(undefined, 8), true],
- ['stale intent annotation', host(undefined, 7, 6), true],
+ // `host(undefined, 7, 6)` — the annotation pinned BEHIND the live
+ // generation — used to belong here. It is now a refusal, not a null, and
+ // has its own block below (#329).
+ ['an intent annotation ahead of the live generation', host(undefined, 7, 8), true],
['invalid annotation', host('caller-value'), true],
['missing intent', host(), false],
// A Host deleted and recreated under the same name reaches the same
@@ -215,4 +218,110 @@ describe('HccAdministrativeOutcomeBindingResolver', () => {
expect(malformed).toBeNull()
expect(witness).toMatchObject({ targetRef: 'mcp-host/chatllm-live', outcome: 'succeeded' })
})
+
+ /**
+ * #329 — the annotation is an obligation nothing ever retires, so once the
+ * Host passes the generation it names, no future report can bind. That is a
+ * different answer from "not yet visible", and the two must not collapse
+ * into the same null.
+ */
+ describe('administrative intent generation drift (#329)', () => {
+ it('refuses an annotation pinned behind the live generation, and counts it', async () => {
+ const metrics = { inc: vi.fn() }
+ const warn = vi.fn()
+ const listResource = vi.fn().mockResolvedValue([host(undefined, 7, 6)])
+ const findHostIntents = vi.fn().mockResolvedValue(intentMap())
+
+ const result = await new HccAdministrativeOutcomeBindingResolver(
+ { getResource: vi.fn(), listResource },
+ { findHostIntent: vi.fn(), findHostIntents },
+ { warn } as never,
+ metrics
+ ).resolve(principal, event)
+
+ expect(result).toEqual({ refusal: 'administrative_intent_generation_drift' })
+ // Liveness witnesses: the refusal is not the resolver skipping the
+ // candidate loop. It read the Hosts, it reached the decision, and it
+ // reported the decision.
+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')
+ expect(metrics.inc).toHaveBeenCalledWith({ namespace: 'mcp-host' })
+ expect(warn).toHaveBeenCalledWith(
+ expect.objectContaining({
+ event: 'administrative_intent_generation_drift',
+ namespace: 'mcp-host',
+ liveGeneration: 7,
+ annotatedGeneration: 6,
+ }),
+ expect.any(String)
+ )
+ })
+
+ /**
+ * The asymmetry is the whole point. `annotated < live` can never bind
+ * again; `annotated > live` is the window between control-api predicting a
+ * generation and the corrective patch landing, and it resolves on its own.
+ * Both rows run through the SAME call, so a resolver that answered one way
+ * for everything would fail.
+ */
+ it('separates a superseded annotation from one that is merely ahead', async () => {
+ const metrics = { inc: vi.fn() }
+ const behind = host(undefined, 7, 6)
+ const ahead = host(undefined, 7, 9)
+ ahead.metadata.name = 'chatllm-ahead'
+
+ const resolver = new HccAdministrativeOutcomeBindingResolver(
+ { getResource: vi.fn(), listResource: vi.fn().mockResolvedValue([behind, ahead]) },
+ {
+ findHostIntent: vi.fn(),
+ findHostIntents: vi
+ .fn()
+ .mockResolvedValue(new Map([...intentMap(), ...intentMap(undefined, 'chatllm-ahead')])),
+ },
+ { warn: vi.fn() } as never,
+ metrics
+ )
+
+ const [superseded, notYet] = await resolver.resolveMany(principal, [
+ event,
+ {
+ ...event,
+ sourceEventId: 'outcome-ahead',
+ sourceStatusRef: `host:mcp-host/chatllm-ahead:generation=7:uid=${HOST_UID}`,
+ },
+ ])
+
+ expect(superseded).toEqual({ refusal: 'administrative_intent_generation_drift' })
+ expect(notYet).toBeNull()
+ // Exactly one of the two was counted: the metric names the terminal case
+ // only, so a dashboard built on it is not inflated by the transient one.
+ expect(metrics.inc).toHaveBeenCalledTimes(1)
+ expect(metrics.inc).toHaveBeenCalledWith({ namespace: 'mcp-host' })
+ })
+
+ /**
+ * Identity is checked before drift. A same-name Host with a different uid
+ * is not the object the reporter observed, so its annotation says nothing
+ * about this event — answering "terminal drift" there would turn a #694
+ * recreation into a permanently dropped outcome.
+ */
+ it('does not report drift when the uid does not match', async () => {
+ const metrics = { inc: vi.fn() }
+ const other = host(undefined, 7, 6, '00000000-0000-4000-8000-000000000000')
+ const listResource = vi.fn().mockResolvedValue([other])
+
+ const result = await new HccAdministrativeOutcomeBindingResolver(
+ { getResource: vi.fn(), listResource },
+ { findHostIntent: vi.fn(), findHostIntents: vi.fn().mockResolvedValue(intentMap()) },
+ { warn: vi.fn() } as never,
+ metrics
+ ).resolve(principal, event)
+
+ expect(result).toBeNull()
+ // Liveness witness for the negative assertion below: the resolver did
+ // run the lookup and reach the candidate loop, so the uncounted metric
+ // is a decision, not an unexecuted path.
+ expect(listResource).toHaveBeenCalledWith('hosts', 'mcp-host')
+ expect(metrics.inc).not.toHaveBeenCalled()
+ })
+ })
})
diff --git a/control-api/test/services.resourceService.test.ts b/control-api/test/services.resourceService.test.ts
index 0223a70fb..5409111fb 100644
--- a/control-api/test/services.resourceService.test.ts
+++ b/control-api/test/services.resourceService.test.ts
@@ -659,3 +659,183 @@ describe('ResourceService Host administrative intent', () => {
expect(persistHostIntent).not.toHaveBeenCalled()
})
})
+
+/**
+ * #329 — the annotation must name the generation the apiserver PERSISTED, not
+ * the one control-api predicted before the write. The Host CRD declares a
+ * status subresource, so `metadata.generation` advances only on spec changes:
+ * a replace whose spec is unchanged leaves the object at N while the
+ * prediction writes N+1, and nothing ever retires that annotation, so the Host
+ * stops binding outcomes permanently.
+ */
+describe('ResourceService administrative intent generation reconciliation (#329)', () => {
+ const GENERATION = 'clerum.io/administrative-intent-generation'
+ const OPERATION_ID = '11111111-1111-4111-8111-111111111111'
+
+ function withIntent() {
+ setAdministrativeOperationService({
+ persistHostIntent: vi.fn().mockResolvedValue({
+ operationId: OPERATION_ID,
+ action: 'update',
+ namespace: 'mcp-host',
+ targetRef: 'mcp-host/host-a',
+ operatorSub: 'admin-1',
+ requestId: 'request-1',
+ }),
+ persistHostOutcome: vi.fn(),
+ } as never)
+ }
+
+ /**
+ * `replaced` is what the apiserver answers the replace with — the object as
+ * persisted. `generation: 4` against a live 4 is the unchanged-spec case.
+ */
+ function api(replaced: Record<string, unknown>, patch = vi.fn().mockResolvedValue({})) {
+ return {
+ getNamespacedCustomObject: vi.fn().mockResolvedValue({
+ metadata: {
+ name: 'host-a',
+ namespace: 'mcp-host',
+ resourceVersion: '5',
+ generation: 4,
+ annotations: {},
+ },
+ spec: { contextRef: 'live' },
+ }),
+ replaceNamespacedCustomObject: vi.fn().mockResolvedValue(replaced),
+ patchNamespacedCustomObject: patch,
+ listNamespacedCustomObject: vi.fn(),
+ }
+ }
+
+ async function update(customApi: ReturnType<typeof api>) {
+ withIntent()
+ const service = new ResourceService(customApi as never, 'control-plane', { hosts: 'mcp-host' })
+ try {
+ return await runWithAdministrativeRequestContext(
+ { operatorSub: 'admin-1', requestId: 'request-1' },
+ () =>
+ service.updateResource('hosts', 'host-a', { spec: { contextRef: 'next' } }, 'mcp-host')
+ )
+ } finally {
+ setAdministrativeOperationService(null)
+ }
+ }
+
+ it('lowers the annotation to the persisted generation when the replace did not bump it', async () => {
+ const patch = vi.fn().mockResolvedValue({})
+ const customApi = api(
+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },
+ patch
+ )
+
+ await update(customApi)
+
+ // Liveness: the replace itself carried the PREDICTED value, so the patch
+ // below is a correction of a real mismatch and not a no-op.
+ const replaceBody = customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body
+ expect(replaceBody.metadata.annotations[GENERATION]).toBe('5')
+
+ expect(patch).toHaveBeenCalledOnce()
+ const [patchArgs] = patch.mock.calls[0]!
+ expect(patchArgs).toMatchObject({
+ namespace: 'mcp-host',
+ plural: 'hosts',
+ name: 'host-a',
+ body: {
+ metadata: {
+ // The precondition comes from the object the write returned, so a
+ // concurrent change between the replace and the patch is refused
+ // rather than overwritten.
+ resourceVersion: '6',
+ annotations: { [GENERATION]: '4' },
+ },
+ },
+ })
+ // Annotations only: a patch that carried a spec would bump the generation
+ // it is correcting and feed itself.
+ expect(Object.keys(patchArgs.body.metadata)).toEqual(['resourceVersion', 'annotations'])
+ expect(patchArgs.body).not.toHaveProperty('spec')
+ })
+
+ it('does not patch when the apiserver assigned the predicted generation', async () => {
+ const patch = vi.fn().mockResolvedValue({})
+ const customApi = api(
+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 5, resourceVersion: '6' } },
+ patch
+ )
+
+ await update(customApi)
+
+ // Liveness witness for the negative assertion: the write really happened
+ // and really annotated 5, so the absent patch is the equality check and
+ // not an unexecuted path.
+ expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()
+ expect(
+ customApi.replaceNamespacedCustomObject.mock.calls[0]![0].body.metadata.annotations[
+ GENERATION
+ ]
+ ).toBe('5')
+ expect(patch).not.toHaveBeenCalled()
+ })
+
+ /**
+ * A 409 means the resourceVersion from the write is stale. The re-read
+ * decides whether the correction is still ours to make: if the object has
+ * moved past the generation THIS operation produced, writing either value
+ * would attribute a generation the intent did not cause, so the method stops
+ * and the binding resolver refuses those outcomes as drift — which is the
+ * correct answer, not a regression.
+ */
+ it('stops correcting when a concurrent write moved the object past our generation', async () => {
+ const conflict = makeConflictError()
+ const patch = vi.fn().mockRejectedValue(conflict)
+ const customApi = api(
+ { metadata: { name: 'host-a', namespace: 'mcp-host', generation: 4, resourceVersion: '6' } },
+ patch
+ )
+ // The re-read sees a generation past the one our replace produced.
+ customApi.getNamespacedCustomObject
+ .mockResolvedValueOnce({
+ metadata: {
+ name: 'host-a',
+ namespace: 'mcp-host',
+ resourceVersion: '5',
+ generation: 4,
+ annotations: {},
+ },
+ spec: { contextRef: 'live' },
+ })
+ .mockResolvedValue({
+ metadata: {
+ name: 'host-a',
+ namespace: 'mcp-host',
+ resourceVersion: '9',
+ generation: 6,
+ annotations: {},
+ },
+ spec: { contextRef: 'other' },
+ })
+
+ await update(customApi)
+
+ // Liveness: one patch WAS attempted and got the 409, so the single call is
+ // the stop rule and not a method that never tried.
+ expect(patch).toHaveBeenCalledOnce()
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalledTimes(2)
+ })
+
+ it('reports rather than throws when the write response carries no generation', async () => {
+ const patch = vi.fn().mockResolvedValue({})
+ const customApi = api({ metadata: { name: 'host-a', namespace: 'mcp-host' } }, patch)
+
+ // The mutation itself succeeded, so it must still return normally: turning
+ // an uncorrectable annotation into a caller-visible error would report a
+ // completed write as failed.
+ await expect(update(customApi)).resolves.toMatchObject({
+ metadata: { name: 'host-a' },
+ })
+ expect(customApi.replaceNamespacedCustomObject).toHaveBeenCalledOnce()
+ expect(patch).not.toHaveBeenCalled()
+ })
+})
diff --git a/control-api/test/services.routeTracingSubmission.test.ts b/control-api/test/services.routeTracingSubmission.test.ts
index f7524533f..a2984946e 100644
--- a/control-api/test/services.routeTracingSubmission.test.ts
+++ b/control-api/test/services.routeTracingSubmission.test.ts
@@ -24,6 +24,7 @@ import type {
TracingTransactionRunner,
} from '../src/services/tracing/contracts.js'
import {
+ AdministrativeIntentGenerationDriftError,
InvalidTracingInputError,
RouteTracingSubmissionService,
TracingBindingUnavailableError,
@@ -196,6 +197,62 @@ describe('route tracing submission facade', () => {
expect(appendManyInTransaction).not.toHaveBeenCalled()
})
+ /**
+ * #329 — the classification trap. `rejectionReason` allowlists error classes
+ * plus statuses [400, 403]; an unlisted 409 falls through to
+ * `submission_failed`, which `operationalStatus.ts` treats as CRITICAL. A
+ * drift refusal is an operator-attributable rejection, not a control-api
+ * malfunction, so mislabelling it would trade a 403 log loop for a critical
+ * alert per drifted Host.
+ */
+ it('classifies a generation-drift refusal as event_rejected, not submission_failed', async () => {
+ const lastError = vi.spyOn(governedTraceLastErrorTimestampSeconds, 'set')
+ const operationalError = vi.spyOn(governedTraceOperationalErrorsTotal, 'inc')
+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
+ const h = transactionHarness()
+ const appendManyInTransaction = vi.fn()
+ const service = new RouteTracingSubmissionService({
+ transaction: h.transaction,
+ administrativeOperationBindingResolver: {
+ resolve: vi.fn(),
+ resolveMany: vi
+ .fn()
+ .mockResolvedValue([{ refusal: 'administrative_intent_generation_drift' }]),
+ },
+ administrativeEventAppender: { appendManyInTransaction },
+ })
+
+ const failure = service.submit({
+ principal: administrativePrincipal,
+ events: [administrativeInput],
+ })
+ await expect(failure).rejects.toBeInstanceOf(AdministrativeIntentGenerationDriftError)
+ await expect(failure).rejects.toMatchObject({
+ code: 'administrative_intent_generation_drift',
+ status: 409,
+ statusCode: 409,
+ eventIndex: 0,
+ })
+
+ // Liveness: the rejection path ran to completion — the event was counted
+ // as rejected and the error clock was stamped.
+ expect(rejected).toHaveBeenCalledOnce()
+ expect(lastError).toHaveBeenCalledWith(
+ { scope: 'administrative', reason: 'event_rejected' },
+ expect.any(Number)
+ )
+ expect(lastError).not.toHaveBeenCalledWith(
+ { scope: 'administrative', reason: 'submission_failed' },
+ expect.any(Number)
+ )
+ expect(operationalError).not.toHaveBeenCalledWith({
+ scope: 'administrative',
+ reason: 'submission_failed',
+ })
+ expect(h.transactionSpy).not.toHaveBeenCalled()
+ expect(appendManyInTransaction).not.toHaveBeenCalled()
+ })
+
it('records accepted and replayed events with only bounded trace labels after commit', async () => {
const accepted = vi.spyOn(governedTraceAcceptedTotal, 'inc')
const replayed = vi.spyOn(governedTraceReplayedTotal, 'inc')
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index afd9bad63..a0a12dbe2 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.346",
+ "version": "0.1.347",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.346",
+ "version": "0.1.347",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 24992abae..2fbf1bd8d 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.346",
+ "version": "0.1.347",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/administrativeOutcomeReporter.test.ts b/host-context-controller/src/administrativeOutcomeReporter.test.ts
index ccf89e93a..6b42c387f 100644
--- a/host-context-controller/src/administrativeOutcomeReporter.test.ts
+++ b/host-context-controller/src/administrativeOutcomeReporter.test.ts
@@ -346,4 +346,50 @@ describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)
expect(submittedIds(fetchFn)).toEqual(['first', 'second', 'third', 'first'])
expect(await counted('deduplicated')).toBe(1)
})
+
+ /**
+ * #329 — the loop this cuts. A Host whose intent annotation names a
+ * generation it has already passed answers 409
+ * `administrative_intent_generation_drift` forever, and before this change
+ * that answer was a plain retryable failure: 3 submits per enqueue, and
+ * `onDrop` deleted the dedupe key so the next reconciler pass enqueued it
+ * again. Measured in dev at ~108 requests/hour for a single drifted Host.
+ *
+ * The terminal classification routes it to `onTerminal`, which PRESERVES the
+ * key, so the re-enqueue is deduplicated instead of resubmitted. The bound
+ * is honest: 1 submit, then 0 while the key stays resident — not zero, and
+ * not permanent, since `seen` is memory and does not survive a restart.
+ */
+ it('settles a 409 generation-drift refusal once and stops re-sending it (#329)', async () => {
+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
+ const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
+ .events[0]!.sourceEventId
+ return id === 'drifted'
+ ? failedResponse(409, {
+ code: 'administrative_intent_generation_drift',
+ error: 'ignored',
+ })
+ : failedResponse(500, { error: 'Internal server error' })
+ }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('drifted'))
+ reporter.enqueueHostOutcome(outcome('transient'))
+ await settle()
+ // The reconciler observes the same drifted Host on its next pass.
+ reporter.enqueueHostOutcome(outcome('drifted'))
+ await settle()
+
+ const ids = submittedIds(fetchFn)
+ // Liveness witness: a retryable failure on another key in the same run
+ // used its full budget of 3, so the single 'drifted' submit is the
+ // terminal classification and not a reporter that stopped submitting.
+ expect(ids.filter(id => id === 'transient')).toHaveLength(3)
+ expect(ids.filter(id => id === 'drifted')).toHaveLength(1)
+ expect(await counted('rejected')).toBe(1)
+ expect(await counted('deduplicated')).toBe(1)
+ // Not a conflict: that label belongs to `tracing_idempotency_conflict`,
+ // and conflating them would hide the drift inside an existing metric.
+ expect(await counted('conflict')).toBe(0)
+ })
})
diff --git a/host-context-controller/src/reporterHttpFailure.test.ts b/host-context-controller/src/reporterHttpFailure.test.ts
index e2af52fc7..a0a2030d5 100644
--- a/host-context-controller/src/reporterHttpFailure.test.ts
+++ b/host-context-controller/src/reporterHttpFailure.test.ts
@@ -34,6 +34,7 @@ describe('throwForFailedSubmit', () => {
it.each([
[409, 'tracing_idempotency_conflict', 'conflict'],
+ [409, 'administrative_intent_generation_drift', 'rejected'],
[400, 'unsafe_tracing_input', 'rejected'],
[400, 'invalid_tracing_input', 'rejected'],
] as const)('treats %i %s as terminal %s', async (status, code, result) => {
@@ -43,11 +44,33 @@ describe('throwForFailedSubmit', () => {
expect(error).toMatchObject({ result })
})
+ /**
+ * Two codes now share status 409 (#329), so the `find` on `code` decides
+ * which terminal result applies rather than merely confirming the only
+ * candidate. A resolver that returned the first 409 entry would answer
+ * `conflict` for a drift refusal, and HCC would record a routine operator
+ * outcome as an idempotency collision.
+ */
+ it('picks the terminal result by code when two entries share status 409', async () => {
+ const conflict = await failure(409, async () => ({ code: 'tracing_idempotency_conflict' }))
+ const drift = await failure(409, async () => ({
+ code: 'administrative_intent_generation_drift',
+ }))
+
+ expect(conflict).toMatchObject({ result: 'conflict' })
+ expect(drift).toMatchObject({ result: 'rejected' })
+ })
+
it.each([
[400, 'tracing_idempotency_conflict'],
[409, 'unsafe_tracing_input'],
[409, 'invalid_tracing_input'],
[403, 'tracing_idempotency_conflict'],
+ // The drift code carried on a 403 is NOT terminal. The 403 next door is
+ // `tracing_binding_unavailable`, which also covers a binding that is
+ // merely not visible yet; keeping the whole status retryable is what lets
+ // a control-api/HCC deploy overlap heal itself (#329).
+ [403, 'administrative_intent_generation_drift'],
] as const)(
'keeps %i with %s retryable: the status is part of the pair',
async (status, code) => {
diff --git a/host-context-controller/src/reporterHttpFailure.ts b/host-context-controller/src/reporterHttpFailure.ts
index a295abd88..043517ba2 100644
--- a/host-context-controller/src/reporterHttpFailure.ts
+++ b/host-context-controller/src/reporterHttpFailure.ts
@@ -12,6 +12,17 @@ const log = hccLogger.child({ module: 'reporter-http-failure' })
* not read (#693) — and neither becomes acceptable on a retry, so it is as
* final as `unsafe_tracing_input`. Every other failure (403 binding not yet
* visible, 5xx, a 4xx without a code, a body that is not JSON) stays retryable.
+ *
+ * `administrative_intent_generation_drift` (#329) is deterministic for a
+ * different reason than the others: the request is well formed and the server
+ * state is real, but the Host carries an intent annotation pinned to a
+ * generation it has already passed, and nothing ever retires that annotation.
+ * Generations only grow and the reporter always reads the live object, so the
+ * answer cannot change. The plain 403 next door is NOT deterministic — it also
+ * covers a binding that is merely not visible yet, which is what lets a
+ * control-api/HCC deploy overlap heal itself — so only the coded 409 is listed
+ * here. Two entries now share status 409, which makes the `find` on `code`
+ * below load-bearing rather than incidental.
*/
const TERMINAL_RESPONSES: ReadonlyArray<{
status: number
@@ -19,6 +30,7 @@ const TERMINAL_RESPONSES: ReadonlyArray<{
result: ReporterTerminalResult
}> = [
{ status: 409, code: 'tracing_idempotency_conflict', result: 'conflict' },
+ { status: 409, code: 'administrative_intent_generation_drift', result: 'rejected' },
{ status: 400, code: 'unsafe_tracing_input', result: 'rejected' },
{ status: 400, code: 'invalid_tracing_input', result: 'rejected' },
]