Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc,control-api): stop tracing idempotency 409s for admin outcomes and recreated Hosts (#692)

ALFREDO J LOPEZ H (alfredolopez80.eth)ccd92ec04d96
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 9837b354e..5d2665b68 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.586",
+ "version": "0.1.587",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.586",
+ "version": "0.1.587",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index ddcea757c..9ac84821c 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.586",
+ "version": "0.1.587",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/http/errorHandler.ts b/control-api/src/http/errorHandler.ts
index 07581b4b2..25f61b94a 100644
--- a/control-api/src/http/errorHandler.ts
+++ b/control-api/src/http/errorHandler.ts
@@ -12,6 +12,16 @@ export const FORWARDABLE_INTEGRATION_CODES = new Set([
'registry_integration_error',
])
+// Stable machine-readable codes added to a forwarded 4xx body. Internal
+// reporters (HCC) use them to tell a deterministic rejection, which must not be
+// retried, from a transient one. Only codes set by our own error constructors
+// belong here; every other 4xx keeps the `{ error, correlationId }` shape.
+export const FORWARDABLE_CLIENT_ERROR_CODES = new Set([
+ 'tracing_idempotency_conflict',
+ 'unsafe_tracing_input',
+ 'invalid_tracing_input',
+])
+
/**
* Extract an HTTP status from an error the same way `extractK8sStatus`
* (resourceServiceHelpers) does, so a raw @kubernetes/client-node error — which
@@ -168,17 +178,24 @@ export function clerumErrorHandler(
const isClientError = errStatus !== undefined && errStatus >= 400 && errStatus < 500
if (isClientError) {
+ const clientCode = (err as { code?: unknown }).code
+ const forwardedCode =
+ typeof clientCode === 'string' && FORWARDABLE_CLIENT_ERROR_CODES.has(clientCode)
+ ? clientCode
+ : undefined
log.warn(
{
event: 'forwarded_client_error',
correlationId,
status: errStatus,
+ ...(forwardedCode ? { code: forwardedCode } : {}),
err: safeErrorLogMessage(err),
},
'forwarded client error from upstream'
)
res.status(errStatus as number).json({
error: clientErrorMessage(err, errStatus as number),
+ ...(forwardedCode ? { code: forwardedCode } : {}),
correlationId,
})
return
diff --git a/control-api/src/services/tracing/administrativeEvents.ts b/control-api/src/services/tracing/administrativeEvents.ts
index 5db1f66e5..bfc2db3d9 100644
--- a/control-api/src/services/tracing/administrativeEvents.ts
+++ b/control-api/src/services/tracing/administrativeEvents.ts
@@ -59,6 +59,13 @@ export class AdministrativePrincipalBindingInvariantError extends Error {
}
}
+function isHccLinkedOutcome(
+ principal: AdministrativeEventSubmitterPrincipalV1 | ControlApiLocalAdministrativePrincipalV1,
+ input: AdministrativeEventInputV1
+): boolean {
+ return principal.kind === 'hcc_internal_control' && input.kind === 'linked_outcome'
+}
+
function assertPrincipalBinding(
principal: AdministrativeEventSubmitterPrincipalV1 | ControlApiLocalAdministrativePrincipalV1,
binding: AdministrativeServerBindingV1,
@@ -195,7 +202,14 @@ export class AdministrativeEventService {
serviceSub: principal.serviceSub,
sourceKind: principal.kind,
sourceEventId: input.sourceEventId,
- occurredAt: occurredAtIso,
+ // HCC linked outcomes are state facts keyed by operation, generation and
+ // outcome. HCC holds no stable observation time (it re-observes the
+ // outcome on every reconcile pass and after every restart), so hashing
+ // occurredAt turned each re-observation into an idempotency conflict
+ // (#327). Storage keeps the first occurredAt; reasonCode, sourceStatusRef,
+ // payload and the server-resolved binding still distinguish a genuinely
+ // different claim. Every other submitter keeps occurredAt in the hash.
+ ...(isHccLinkedOutcome(principal, input) ? {} : { occurredAt: occurredAtIso }),
kind: input.kind,
reasonCode: input.reasonCode ?? null,
sourceStatusRef: input.sourceStatusRef ?? null,
diff --git a/control-api/src/services/tracing/append.ts b/control-api/src/services/tracing/append.ts
index 668cf5932..c8ed51f75 100644
--- a/control-api/src/services/tracing/append.ts
+++ b/control-api/src/services/tracing/append.ts
@@ -100,13 +100,35 @@ export class TracingIdempotencyConflictError extends Error {
}
}
+/**
+ * The code stays `unsafe_tracing_input` for every reason; only the message
+ * differs. The server-owned/monetary message used to cover all three, which
+ * misdescribed a field such as `gfs_subject` as authority smuggling (#328).
+ * - `not_permitted`: a field or payload key outside the allowlist.
+ * - `invalid_value`: an allowlisted key whose value has the wrong type,
+ * format or size.
+ */
+export type UnsafeTracingInputReason =
+ | 'server_owned_or_monetary'
+ | 'not_permitted'
+ | 'invalid_value'
+
+const UNSAFE_TRACING_INPUT_MESSAGES: Record<UnsafeTracingInputReason, string> = {
+ server_owned_or_monetary: 'tracing input contains a server-owned or monetary field',
+ not_permitted: 'tracing input field is not permitted',
+ invalid_value: 'tracing input value is not accepted',
+}
+
export class UnsafeTracingInputError extends Error {
readonly code = 'unsafe_tracing_input'
readonly status = 400
readonly statusCode = 400
- constructor(readonly field: string) {
- super(`tracing input contains a server-owned or monetary field: ${field}`)
+ constructor(
+ readonly field: string,
+ readonly reason: UnsafeTracingInputReason = 'server_owned_or_monetary'
+ ) {
+ super(`${UNSAFE_TRACING_INPUT_MESSAGES[reason]}: ${field}`)
this.name = 'UnsafeTracingInputError'
}
}
@@ -233,39 +255,41 @@ export function assertNoClientAuthority(value: unknown, path = 'input'): void {
export function assertSafeEventPayload(value: unknown): void {
if (value === undefined) return
if (!value || typeof value !== 'object' || Array.isArray(value)) {
- throw new UnsafeTracingInputError('input.payload')
+ throw new UnsafeTracingInputError('input.payload', 'invalid_value')
}
for (const [key, fieldValue] of Object.entries(value as Record<string, unknown>)) {
+ if (!SAFE_METADATA_KEYS.has(key)) {
+ throw new UnsafeTracingInputError(`input.payload.${key}`, 'not_permitted')
+ }
if (
- !SAFE_METADATA_KEYS.has(key) ||
!['string', 'number'].includes(typeof fieldValue) ||
(typeof fieldValue === 'number' && !Number.isFinite(fieldValue))
) {
- throw new UnsafeTracingInputError(`input.payload.${key}`)
+ throw new UnsafeTracingInputError(`input.payload.${key}`, 'invalid_value')
}
if (
key === 'tool_kind' &&
!['internal_tool', 'mcp_server_tool', 'workflow'].includes(String(fieldValue))
) {
- throw new UnsafeTracingInputError('input.payload.tool_kind')
+ throw new UnsafeTracingInputError('input.payload.tool_kind', 'invalid_value')
}
if (
key === 'tool_source_ref' &&
(typeof fieldValue !== 'string' || fieldValue.length === 0 || fieldValue.length > 128)
) {
- throw new UnsafeTracingInputError('input.payload.tool_source_ref')
+ throw new UnsafeTracingInputError('input.payload.tool_source_ref', 'invalid_value')
}
if (
key === 'target_label' &&
(typeof fieldValue !== 'string' || !/^[A-Za-z0-9._-]{3,64}$/.test(fieldValue))
) {
- throw new UnsafeTracingInputError('input.payload.target_label')
+ throw new UnsafeTracingInputError('input.payload.target_label', 'invalid_value')
}
if (
key === 'target_principal_kind' &&
!['operator', 'host', 'context', 'service'].includes(String(fieldValue))
) {
- throw new UnsafeTracingInputError('input.payload.target_principal_kind')
+ throw new UnsafeTracingInputError('input.payload.target_principal_kind', 'invalid_value')
}
if (
key === 'target_principal_ref' &&
@@ -274,14 +298,14 @@ export function assertSafeEventPayload(value: unknown): void {
fieldValue.length > 256 ||
!/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(fieldValue))
) {
- throw new UnsafeTracingInputError('input.payload.target_principal_ref')
+ throw new UnsafeTracingInputError('input.payload.target_principal_ref', 'invalid_value')
}
}
const payload = value as Record<string, unknown>
const targetPrincipalKind = payload.target_principal_kind
const targetPrincipalRef = payload.target_principal_ref
if ((targetPrincipalKind === undefined) !== (targetPrincipalRef === undefined)) {
- throw new UnsafeTracingInputError('input.payload.target_principal')
+ throw new UnsafeTracingInputError('input.payload.target_principal', 'invalid_value')
}
if (typeof targetPrincipalKind === 'string' && typeof targetPrincipalRef === 'string') {
const hasCanonicalPrefix =
@@ -289,11 +313,11 @@ export function assertSafeEventPayload(value: unknown): void {
(targetPrincipalKind !== 'operator' &&
targetPrincipalRef.startsWith(`${targetPrincipalKind}:`))
if (!hasCanonicalPrefix) {
- throw new UnsafeTracingInputError('input.payload.target_principal_ref')
+ throw new UnsafeTracingInputError('input.payload.target_principal_ref', 'invalid_value')
}
}
if (Buffer.byteLength(JSON.stringify(value), 'utf8') > 16_384) {
- throw new UnsafeTracingInputError('input.payload')
+ throw new UnsafeTracingInputError('input.payload', 'invalid_value')
}
}
diff --git a/control-api/src/services/tracing/contracts.ts b/control-api/src/services/tracing/contracts.ts
index 49a8b606f..bd2edd30e 100644
--- a/control-api/src/services/tracing/contracts.ts
+++ b/control-api/src/services/tracing/contracts.ts
@@ -168,6 +168,8 @@ export interface InfrastructureTelemetryEventInputV1 extends SourceOccurrenceInp
name: string
namespace: string
generation?: number
+ /** metadata.uid of the observed Host; a recreated Host with the same name must not bind. */
+ uid?: string
}
/** Non-authoritative lookup hint; control-api derives the WorkflowRecipe binding. */
workflowRunLookupReference?: { runId: string }
diff --git a/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts b/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts
index 9ee62c257..a0da0b6d0 100644
--- a/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts
+++ b/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts
@@ -90,7 +90,11 @@ export class HccHealthTransitionBindingResolver implements InfrastructureWorkloa
host.kind !== 'Host' ||
metadata?.name !== reference.name ||
metadata.namespace !== reference.namespace ||
- (reference.generation !== undefined && metadata.generation !== reference.generation)
+ (reference.generation !== undefined && metadata.generation !== reference.generation) ||
+ // A Host deleted and recreated under the same name restarts at generation 1;
+ // only the uid tells the objects apart, so an event observed on the old
+ // object must not bind to its successor (#691).
+ (reference.uid !== undefined && metadata.uid !== reference.uid)
) {
return null
}
diff --git a/control-api/src/services/tracing/routeSubmissionService.ts b/control-api/src/services/tracing/routeSubmissionService.ts
index 5a34e6917..19b7cc5f1 100644
--- a/control-api/src/services/tracing/routeSubmissionService.ts
+++ b/control-api/src/services/tracing/routeSubmissionService.ts
@@ -190,7 +190,9 @@ function validateShape(
`events[${index}]`
)
for (const field of Object.keys(event)) {
- if (!allowed.has(field)) throw new UnsafeTracingInputError(`events[${index}].${field}`)
+ if (!allowed.has(field)) {
+ throw new UnsafeTracingInputError(`events[${index}].${field}`, 'not_permitted')
+ }
}
}
@@ -301,7 +303,10 @@ function normalizeInfra(
typeof (hostLookupReference as Record<string, unknown>).name !== 'string' ||
typeof (hostLookupReference as Record<string, unknown>).namespace !== 'string' ||
((hostLookupReference as Record<string, unknown>).generation !== undefined &&
- typeof (hostLookupReference as Record<string, unknown>).generation !== 'number'))
+ typeof (hostLookupReference as Record<string, unknown>).generation !== 'number') ||
+ ((hostLookupReference as Record<string, unknown>).uid !== undefined &&
+ (typeof (hostLookupReference as Record<string, unknown>).uid !== 'string' ||
+ (hostLookupReference as Record<string, unknown>).uid === '')))
) {
throw new InvalidTracingInputError(
`events[${index}].hostLookupReference must be a Host lookup reference`
diff --git a/control-api/test/http.errorHandler.clientErrorCode.test.ts b/control-api/test/http.errorHandler.clientErrorCode.test.ts
new file mode 100644
index 000000000..09731b17d
--- /dev/null
+++ b/control-api/test/http.errorHandler.clientErrorCode.test.ts
@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import express from 'express'
+import request from 'supertest'
+import { clerumErrorHandler } from '../src/http/errorHandler.js'
+import {
+ TracingIdempotencyConflictError,
+ UnsafeTracingInputError,
+} from '../src/services/tracing/append.js'
+import { GovernedReadInvalidQueryError } from '../src/services/tracing/governedEventReadService.js'
+import {
+ InvalidTracingInputError,
+ TracingBindingUnavailableError,
+} from '../src/services/tracing/routeSubmissionService.js'
+
+/**
+ * Internal reporters (HCC) must tell a deterministic tracing rejection, which
+ * they must not retry, from a transient one (#326/#327). The global handler
+ * exposes a stable `code` for an allowlist of our own error types only.
+ */
+function appThrowing(err: unknown) {
+ const app = express()
+ app.post('/boom', (_req, _res, next) => next(err))
+ app.use(clerumErrorHandler)
+ return app
+}
+
+describe('clerumErrorHandler — machine-readable code on allowlisted 4xx', () => {
+ it.each([
+ [
+ new TracingIdempotencyConflictError('administrative', 'hcc_internal_control', 'e-1'),
+ 409,
+ 'tracing_idempotency_conflict',
+ ],
+ [new UnsafeTracingInputError('payload.gfs_subject'), 400, 'unsafe_tracing_input'],
+ [new InvalidTracingInputError('events[0].kind is not supported'), 400, 'invalid_tracing_input'],
+ ])('forwards %s with its code', async (err, status, code) => {
+ const res = await request(appThrowing(err)).post('/boom')
+
+ expect(res.status).toBe(status)
+ expect(res.body).toMatchObject({ code, error: expect.any(String) })
+ expect(res.body.correlationId).toEqual(expect.any(String))
+ })
+
+ it.each([
+ [new TracingBindingUnavailableError('operation', 0), 403],
+ [new GovernedReadInvalidQueryError('cursor is not valid'), 400],
+ [{ code: 409 }, 409],
+ ])('keeps the body without code for a non-allowlisted 4xx (%s)', async (err, status) => {
+ const res = await request(appThrowing(err)).post('/boom')
+
+ // Liveness: the 4xx branch ran (status forwarded, correlation id present).
+ expect(res.status).toBe(status)
+ expect(res.body.correlationId).toEqual(expect.any(String))
+ expect(res.body).not.toHaveProperty('code')
+ })
+})
diff --git a/control-api/test/routes.tracingSubmissionBoundary.test.ts b/control-api/test/routes.tracingSubmissionBoundary.test.ts
index ad56b1b28..24ff23150 100644
--- a/control-api/test/routes.tracingSubmissionBoundary.test.ts
+++ b/control-api/test/routes.tracingSubmissionBoundary.test.ts
@@ -3,9 +3,18 @@ import express from 'express'
import jwt from 'jsonwebtoken'
import request from 'supertest'
import { config } from '../src/config.js'
+import { clerumErrorHandler } from '../src/http/errorHandler.js'
import { createInternalAdministrativeEventsRouter } from '../src/routes/internal/administrativeEvents.js'
import { createInternalAgentRunEventsRouter } from '../src/routes/internal/agentRunEvents.js'
import { createInternalInfrastructureTelemetryEventsRouter } from '../src/routes/internal/infrastructureTelemetryEvents.js'
+import {
+ TracingIdempotencyConflictError,
+ UnsafeTracingInputError,
+} from '../src/services/tracing/append.js'
+import {
+ InvalidTracingInputError,
+ TracingBindingUnavailableError,
+} from '../src/services/tracing/routeSubmissionService.js'
import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'
function signInternalControl(issuer: 'hcc' | 'wrc'): string {
@@ -283,3 +292,72 @@ describe('internal tracing submission routers', () => {
expect(service.submit).not.toHaveBeenCalled()
})
})
+
+// HCC settles a submission without retry only on these (status, code) pairs
+// (#326), so the code has to survive the real router and the global handler.
+describe('internal tracing submission routers — rejection code on the wire', () => {
+ type RejectingService = { submit: () => Promise<never> }
+ function rejectingApp(router: (service: RejectingService) => express.Router, err: unknown) {
+ const service = { submit: vi.fn<() => Promise<never>>(async () => Promise.reject(err)) }
+ const app = express()
+ app.use(router(service))
+ app.use(clerumErrorHandler)
+ return { app, service }
+ }
+
+ it.each([
+ [
+ '/internal/tracing/administrative-events',
+ createInternalAdministrativeEventsRouter,
+ { kind: 'linked_outcome', operationId: 'operation-1' },
+ new TracingIdempotencyConflictError('administrative', 'hcc_internal_control', 'e-1'),
+ 409,
+ 'tracing_idempotency_conflict',
+ ],
+ [
+ '/internal/tracing/infrastructure-telemetry-events',
+ createInternalInfrastructureTelemetryEventsRouter,
+ { telemetryType: 'reconcile_outcome', sourceEventId: 'reconcile-1' },
+ new UnsafeTracingInputError('input.payload.gfs_subject', 'not_permitted'),
+ 400,
+ 'unsafe_tracing_input',
+ ],
+ [
+ '/internal/tracing/infrastructure-telemetry-events',
+ createInternalInfrastructureTelemetryEventsRouter,
+ { telemetryType: 'reconcile_outcome', sourceEventId: 'reconcile-1' },
+ new InvalidTracingInputError('events[0].hostLookupReference.uid must be a string'),
+ 400,
+ 'invalid_tracing_input',
+ ],
+ ] as const)('%s answers %#: status and code', async (path, router, event, err, status, code) => {
+ const { app, service } = rejectingApp(router, err)
+
+ const response = await request(app)
+ .post(path)
+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)
+ .send({ events: [event] })
+
+ expect(service.submit).toHaveBeenCalledOnce()
+ expect(response.status).toBe(status)
+ expect(response.body).toMatchObject({ code, correlationId: expect.any(String) })
+ })
+
+ it('keeps a binding 403 without a code, so HCC keeps retrying it', async () => {
+ const { app, service } = rejectingApp(
+ createInternalAdministrativeEventsRouter,
+ new TracingBindingUnavailableError('operation', 0)
+ )
+
+ const response = await request(app)
+ .post('/internal/tracing/administrative-events')
+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)
+ .send({ events: [{ kind: 'linked_outcome', operationId: 'operation-1' }] })
+
+ // Liveness: the service rejected and the handler's 4xx branch answered.
+ expect(service.submit).toHaveBeenCalledOnce()
+ expect(response.status).toBe(403)
+ expect(response.body.correlationId).toEqual(expect.any(String))
+ expect(response.body).not.toHaveProperty('code')
+ })
+})
diff --git a/control-api/test/services.firstVerticalBindingResolvers.test.ts b/control-api/test/services.firstVerticalBindingResolvers.test.ts
index b1a475f11..78047388c 100644
--- a/control-api/test/services.firstVerticalBindingResolvers.test.ts
+++ b/control-api/test/services.firstVerticalBindingResolvers.test.ts
@@ -310,6 +310,50 @@ describe('first vertical trusted binding resolvers', () => {
).rejects.toMatchObject({ code: HCC_HEALTH_TRANSITION_BINDING_BLOCKER, status: 403 })
})
+ it('binds a reconcile outcome only to the Host object whose uid it observed (#691)', async () => {
+ // Same name, namespace and generation 1: a recreated Host differs only by uid.
+ const hostLookup = {
+ getResource: vi.fn().mockResolvedValue({
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'Host',
+ metadata: { name: 'chatllm', namespace: 'mcp-host', uid: 'new-uid', generation: 1 },
+ }),
+ }
+ const principal = {
+ kind: 'hcc_internal_control',
+ sourceService: 'host-context-controller',
+ serviceSub: 'hcc-provisioner',
+ credentialId: 'hcc-1',
+ resourceAuthority: 'hcc_managed',
+ allowedTelemetryTypes: ['reconcile_outcome'],
+ } as const
+ const resolver = new HccHealthTransitionBindingResolver(hostLookup)
+ const event = (uid?: string) => ({
+ sourceEventId: `reconcile-${uid ?? 'no-uid'}`,
+ occurredAt: NOW,
+ telemetryType: 'reconcile_outcome' as const,
+ hostLookupReference: {
+ name: 'chatllm',
+ namespace: 'mcp-host',
+ generation: 1,
+ ...(uid === undefined ? {} : { uid }),
+ },
+ payload: { status: 'succeeded', reason_code: 'ready' },
+ })
+
+ await expect(resolver.resolve(principal, event('new-uid'))).resolves.toMatchObject({
+ kubernetesUid: 'new-uid',
+ metadataGeneration: 1,
+ })
+ await expect(resolver.resolve(principal, event('old-uid'))).resolves.toBeNull()
+ // A reference without uid (HCC before this change) still binds.
+ await expect(resolver.resolve(principal, event())).resolves.toMatchObject({
+ kubernetesUid: 'new-uid',
+ })
+ // Liveness: the authoritative Host read ran for all three references.
+ expect(hostLookup.getResource).toHaveBeenCalledTimes(3)
+ })
+
it('resolves WRC infrastructure batches from authoritative workflow rows in one query', async () => {
const query = vi.fn().mockResolvedValue({
rows: [
diff --git a/control-api/test/services.routeTracingSubmission.test.ts b/control-api/test/services.routeTracingSubmission.test.ts
index 35869ea49..f7524533f 100644
--- a/control-api/test/services.routeTracingSubmission.test.ts
+++ b/control-api/test/services.routeTracingSubmission.test.ts
@@ -24,10 +24,18 @@ import type {
TracingTransactionRunner,
} from '../src/services/tracing/contracts.js'
import {
+ InvalidTracingInputError,
RouteTracingSubmissionService,
TracingBindingUnavailableError,
} from '../src/services/tracing/routeSubmissionService.js'
-import { NOW, adminBinding, agentBinding, agentInput } from './services.tracingFixtures.js'
+import {
+ NOW,
+ adminBinding,
+ agentBinding,
+ agentInput,
+ infraBinding,
+ infraPrincipal,
+} from './services.tracingFixtures.js'
const workflowPrincipal: AgentRunEventSubmitterPrincipalV1 = {
kind: 'wrc_internal_control',
@@ -366,4 +374,84 @@ describe('route tracing submission facade', () => {
expect.any(Number)
)
})
+
+ it.each([
+ ['accepts', 'host-uid-1', 1],
+ ['rejects', 42, 0],
+ ['rejects', '', 0],
+ ] as const)(
+ '%s a Host lookup reference uid of %o (#691)',
+ async (_verdict, uid, expectedResolves) => {
+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
+ const h = transactionHarness()
+ const resolve = vi.fn().mockResolvedValue(infraBinding)
+ const appendManyInTransaction = vi.fn().mockResolvedValue([appendResult('accepted')])
+ const service = new RouteTracingSubmissionService({
+ transaction: h.transaction,
+ infrastructureWorkloadBindingResolver: { resolve },
+ infrastructureTelemetryAppender: { appendManyInTransaction },
+ })
+ const submission = service.submit({
+ principal: infraPrincipal,
+ events: [
+ {
+ sourceEventId: 'reconcile-1',
+ occurredAt: NOW,
+ telemetryType: 'reconcile_outcome',
+ hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1, uid },
+ payload: { status: 'succeeded', reason_code: 'ready' },
+ },
+ ],
+ })
+
+ if (expectedResolves === 1) {
+ await expect(submission).resolves.toEqual({ accepted: 1, replayed: 0 })
+ expect(resolve.mock.calls[0]![1].hostLookupReference).toEqual({
+ name: 'chatllm',
+ namespace: 'mcp-host',
+ generation: 1,
+ uid,
+ })
+ } else {
+ await expect(submission).rejects.toBeInstanceOf(InvalidTracingInputError)
+ // Liveness: the rejection was counted, so validation ran on this event.
+ expect(rejected).toHaveBeenCalled()
+ }
+ expect(resolve).toHaveBeenCalledTimes(expectedResolves)
+ }
+ )
+
+ it('names an unknown event field as not permitted, not as server-owned (#328)', async () => {
+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')
+ const h = transactionHarness()
+ const resolve = vi.fn().mockResolvedValue(infraBinding)
+ const service = new RouteTracingSubmissionService({
+ transaction: h.transaction,
+ infrastructureWorkloadBindingResolver: { resolve },
+ infrastructureTelemetryAppender: { appendManyInTransaction: vi.fn() },
+ })
+
+ await expect(
+ service.submit({
+ principal: infraPrincipal,
+ events: [
+ {
+ sourceEventId: 'reconcile-1',
+ occurredAt: NOW,
+ telemetryType: 'reconcile_outcome',
+ hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1 },
+ payload: { status: 'succeeded', reason_code: 'ready' },
+ gfsEvidence: 'rotated',
+ },
+ ],
+ })
+ ).rejects.toMatchObject({
+ code: 'unsafe_tracing_input',
+ reason: 'not_permitted',
+ message: 'tracing input field is not permitted: events[0].gfsEvidence',
+ })
+ // Liveness: the rejection was counted, so validation ran on this event.
+ expect(rejected).toHaveBeenCalled()
+ expect(resolve).not.toHaveBeenCalled()
+ })
})
diff --git a/control-api/test/services.tracingAppends.test.ts b/control-api/test/services.tracingAppends.test.ts
index 46d5731cc..291bbf150 100644
--- a/control-api/test/services.tracingAppends.test.ts
+++ b/control-api/test/services.tracingAppends.test.ts
@@ -16,6 +16,7 @@ import {
import {
TracingIdempotencyConflictError,
UnsafeTracingInputError,
+ assertNoClientAuthority,
assertSafeEventPayload,
} from '../src/services/tracing/append.js'
import type {
@@ -369,6 +370,75 @@ describe('governed append idempotency', () => {
).resolves.toMatchObject({ kind: 'replayed' })
})
+ describe('HCC linked outcomes re-observed at a later time (#327)', () => {
+ const hccAdminPrincipal = {
+ kind: 'hcc_internal_control',
+ sourceService: 'host-context-controller',
+ serviceSub: 'hcc-provisioner',
+ credentialId: 'hcc-admin',
+ allowedKinds: ['linked_outcome'],
+ } as const satisfies AdministrativeEventSubmitterPrincipalV1
+ const hccOutcome = {
+ kind: 'linked_outcome' as const,
+ sourceEventId: `hcc-admin-outcome:${adminBinding.operationId}:3:succeeded`,
+ occurredAt: '2026-07-10T09:59:59.000Z',
+ reasonCode: 'reconciled',
+ sourceStatusRef: 'host:mcp-host/chatllm:generation=3',
+ payload: { resource_class: 'Host', status: 'succeeded' },
+ }
+ const service = () =>
+ new AdministrativeEventService({
+ transaction: h.transaction,
+ now: () => new Date(NOW),
+ newEventId: () => EVENT_ID,
+ })
+ const insertCalls = () =>
+ h.query.mock.calls.filter(call => String(call[0]).includes('inserted_family'))
+
+ it('replays the same outcome with a different occurredAt and keeps the first one', async () => {
+ await expect(
+ service().append(hccAdminPrincipal, adminBinding, hccOutcome)
+ ).resolves.toMatchObject({ kind: 'accepted', accepted: 1 })
+ await expect(
+ service().append(hccAdminPrincipal, adminBinding, {
+ ...hccOutcome,
+ occurredAt: '2026-07-10T10:05:00.000Z',
+ })
+ ).resolves.toMatchObject({ kind: 'replayed', replayed: 1, eventId: EVENT_ID })
+
+ const inserts = insertCalls()
+ expect(inserts).toHaveLength(1)
+ expect(JSON.stringify(inserts[0][1])).toContain(hccOutcome.occurredAt)
+ })
+
+ it('still conflicts when the same outcome key claims a different reasonCode', async () => {
+ await service().append(hccAdminPrincipal, adminBinding, hccOutcome)
+
+ await expect(
+ service().append(hccAdminPrincipal, adminBinding, {
+ ...hccOutcome,
+ occurredAt: '2026-07-10T10:05:00.000Z',
+ reasonCode: 'reconcile_exception',
+ })
+ ).rejects.toBeInstanceOf(TracingIdempotencyConflictError)
+ })
+
+ it('keeps occurredAt in the digest for every other linked_outcome submitter', async () => {
+ const wrcOutcome = { ...hccOutcome, sourceEventId: 'wrc-outcome-1' }
+ await expect(
+ service().append(adminPrincipal, adminBinding, wrcOutcome)
+ ).resolves.toMatchObject({ kind: 'accepted' })
+
+ await expect(
+ service().append(adminPrincipal, adminBinding, {
+ ...wrcOutcome,
+ occurredAt: '2026-07-10T10:05:00.000Z',
+ })
+ ).rejects.toBeInstanceOf(TracingIdempotencyConflictError)
+ expect(insertCalls()).toHaveLength(1)
+ })
+ })
+
it('does not collide events from different source services with the same kind and source event id', async () => {
const secondServicePrincipal = { ...agentPrincipal, sourceService: 'mcp-host-secondary' }
const lockIdentities: string[] = []
@@ -457,6 +527,51 @@ describe('governed append idempotency', () => {
})
describe('server-owned tracing fields', () => {
+ it('accepts HCC GFS evidence as a transition and names a non-allowlisted key as such (#328)', () => {
+ expect(() =>
+ assertSafeEventPayload({
+ resource_class: 'Host',
+ reason_code: 'ready',
+ status: 'succeeded',
+ phase: 'deployed',
+ state: 'ready',
+ transition: 'gfs_token:reused',
+ })
+ ).not.toThrow()
+ expect(() =>
+ assertSafeEventPayload({
+ transition: 'gfs_token:reused',
+ gfs_subject: 'host:1st:mcp-host/chatllm',
+ })
+ ).toThrow(
+ expect.objectContaining({
+ code: 'unsafe_tracing_input',
+ reason: 'not_permitted',
+ message: 'tracing input field is not permitted: input.payload.gfs_subject',
+ })
+ )
+ expect(() => assertSafeEventPayload({ transition: { outcome: 'reused' } })).toThrow(
+ expect.objectContaining({
+ code: 'unsafe_tracing_input',
+ reason: 'invalid_value',
+ message: 'tracing input value is not accepted: input.payload.transition',
+ })
+ )
+ expect(() => assertSafeEventPayload({ target_label: 'Label with spaces' })).toThrow(
+ expect.objectContaining({
+ reason: 'invalid_value',
+ message: 'tracing input value is not accepted: input.payload.target_label',
+ })
+ )
+ expect(() => assertNoClientAuthority({ payload: { sourceService: 'hcc' } })).toThrow(
+ expect.objectContaining({
+ code: 'unsafe_tracing_input',
+ message:
+ 'tracing input contains a server-owned or monetary field: input.payload.sourceService',
+ })
+ )
+ })
+
it('accepts a bounded administrator label but rejects arbitrary display text', () => {
expect(() => assertSafeEventPayload({ target_label: 'deleted_admin' })).not.toThrow()
expect(() => assertSafeEventPayload({ target_label: 'Deleted admin with spaces' })).toThrow(
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index ac10a721e..b2905694f 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.340",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.340",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 921d081c5..81dfb1b9c 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.340",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/administrativeOutcomeReporter.test.ts b/host-context-controller/src/administrativeOutcomeReporter.test.ts
index e9e09e8f6..7aaf39516 100644
--- a/host-context-controller/src/administrativeOutcomeReporter.test.ts
+++ b/host-context-controller/src/administrativeOutcomeReporter.test.ts
@@ -1,8 +1,10 @@
-import { describe, expect, it, vi } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
import {
+ type AdministrativeHostOutcomeProjection,
BoundedAdministrativeOutcomeReporter,
createAdministrativeOutcomeReporter,
} from './administrativeOutcomeReporter'
+import { administrativeOutcomeReporterTotal } from './metrics'
describe('createAdministrativeOutcomeReporter', () => {
it('does not construct a reporter when governed tracing is disabled', () => {
@@ -121,3 +123,208 @@ describe('BoundedAdministrativeOutcomeReporter', () => {
expect(fetchFn).toHaveBeenCalledOnce()
})
})
+
+function outcome(sourceEventId: string): AdministrativeHostOutcomeProjection {
+ return {
+ sourceEventId,
+ occurredAt: '2026-09-18T10:00:00.000Z',
+ hostRef: { name: 'chatllm', namespace: 'mcp-host', generation: 7 },
+ outcome: 'succeeded',
+ reasonCode: 'reconciled',
+ }
+}
+
+function failedResponse(status: number, body?: unknown): Response {
+ return {
+ ok: false,
+ status,
+ json: async () => {
+ if (body === undefined) throw new SyntaxError('Unexpected end of JSON input')
+ return body
+ },
+ } as unknown as Response
+}
+
+async function counted(result: string): Promise<number> {
+ const metric = await administrativeOutcomeReporterTotal.get()
+ return metric.values.find(value => value.labels.result === result)?.value ?? 0
+}
+
+function submittedIds(fetchFn: typeof fetch): string[] {
+ return vi.mocked(fetchFn).mock.calls.map(call => {
+ const body = JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> }
+ return body.events[0]!.sourceEventId
+ })
+}
+
+// Default retry delays with random() = 0 are 25 ms then 50 ms.
+const settle = () => new Promise(resolve => setTimeout(resolve, 150))
+
+describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)', () => {
+ beforeEach(() => administrativeOutcomeReporterTotal.reset())
+
+ function reporterWith(
+ fetchFn: typeof fetch,
+ bounds: { capacity?: number; dedupeCapacity?: number } = {}
+ ) {
+ return new BoundedAdministrativeOutcomeReporter({
+ baseUrl: 'http://control-api.test:8090',
+ signToken: () => 'signed',
+ fetchFn,
+ random: () => 0,
+ ...bounds,
+ })
+ }
+
+ it('deduplicates a key while its first copy is queued or waiting for a retry', async () => {
+ const fetchFn = vi.fn().mockResolvedValue(failedResponse(500)) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('in-flight'))
+ reporter.enqueueHostOutcome(outcome('in-flight'))
+ // The first attempt has failed; the retry fires at 25 ms.
+ await new Promise(resolve => setTimeout(resolve, 10))
+ expect(fetchFn).toHaveBeenCalledOnce()
+ reporter.enqueueHostOutcome(outcome('in-flight'))
+ await settle()
+
+ // Liveness: the one queued copy used its whole retry budget.
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ expect(await counted('retry_exhausted')).toBe(1)
+ expect(await counted('deduplicated')).toBe(2)
+ })
+
+ it('sends an outcome again after it was dropped because the buffer was full', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })
+
+ reporter.enqueueHostOutcome(outcome('buffered'))
+ reporter.enqueueHostOutcome(outcome('dropped'))
+ await settle()
+ expect(await counted('buffer_full')).toBe(1)
+ reporter.enqueueHostOutcome(outcome('dropped'))
+ await settle()
+
+ expect(submittedIds(fetchFn)).toEqual(['buffered', 'dropped'])
+ expect(await counted('deduplicated')).toBe(0)
+ })
+
+ it('keeps a key that is seen on every pass ahead of keys seen once', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })
+
+ for (const id of ['repeated', 'once', 'repeated', 'newer', 'repeated']) {
+ reporter.enqueueHostOutcome(outcome(id))
+ await settle()
+ }
+
+ // 'newer' evicted 'once', not 'repeated', because 'repeated' was seen again.
+ expect(submittedIds(fetchFn)).toEqual(['repeated', 'once', 'newer'])
+ expect(await counted('deduplicated')).toBe(2)
+ })
+
+ it('requires the dedupe bound to exceed the queue capacity', () => {
+ const fetchFn = vi.fn() as unknown as typeof fetch
+
+ expect(() => reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 4 })).toThrow(
+ 'dedupeCapacity must be an integer greater than capacity'
+ )
+ expect(reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 5 })).toBeInstanceOf(
+ BoundedAdministrativeOutcomeReporter
+ )
+ })
+
+ it('sends an accepted outcome once however often the reconciler re-observes it', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))
+ await settle()
+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))
+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))
+ await settle()
+
+ expect(fetchFn).toHaveBeenCalledOnce()
+ expect(await counted('accepted')).toBe(1)
+ expect(await counted('deduplicated')).toBe(2)
+ })
+
+ it('settles a 409 idempotency conflict without retry and deduplicates the key afterwards', async () => {
+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
+ const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
+ .events[0]!.sourceEventId
+ return id === 'conflicting'
+ ? failedResponse(409, { code: 'tracing_idempotency_conflict', error: 'conflict' })
+ : failedResponse(500, { error: 'Internal server error' })
+ }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('conflicting'))
+ reporter.enqueueHostOutcome(outcome('transient'))
+ await settle()
+ reporter.enqueueHostOutcome(outcome('conflicting'))
+ await settle()
+
+ const ids = submittedIds(fetchFn)
+ // Liveness: the transient failure on another key is retried twice.
+ expect(ids.filter(id => id === 'transient')).toHaveLength(3)
+ expect(ids.filter(id => id === 'conflicting')).toHaveLength(1)
+ expect(await counted('conflict')).toBe(1)
+ expect(await counted('deduplicated')).toBe(1)
+ })
+
+ it('sends an outcome again after its retries were exhausted', async () => {
+ const fetchFn = vi
+ .fn()
+ .mockResolvedValueOnce(failedResponse(500))
+ .mockResolvedValueOnce(failedResponse(500))
+ .mockResolvedValueOnce(failedResponse(500))
+ .mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:failed'))
+ await settle()
+ expect(await counted('retry_exhausted')).toBe(1)
+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:failed'))
+ await settle()
+
+ expect(fetchFn).toHaveBeenCalledTimes(4)
+ expect(await counted('accepted')).toBe(1)
+ expect(await counted('deduplicated')).toBe(0)
+ })
+
+ it.each([
+ ['a 403 binding not yet visible', failedResponse(403, { error: 'Forbidden' })],
+ ['a 409 without a body', failedResponse(409)],
+ ['a 409 without a code', failedResponse(409, { error: 'Conflict' })],
+ ['a 400 without a code', failedResponse(400, { error: 'Bad Request' })],
+ ])('keeps retrying %s', async (_label, response) => {
+ const fetchFn = vi.fn().mockResolvedValue(response) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn)
+
+ reporter.enqueueHostOutcome(outcome('retryable'))
+ await settle()
+
+ expect(fetchFn).toHaveBeenCalledTimes(3)
+ expect(await counted('retry_exhausted')).toBe(1)
+ expect(await counted('conflict')).toBe(0)
+ expect(await counted('rejected')).toBe(0)
+ })
+
+ it('forgets the oldest settled outcome once the dedupe bound is reached', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })
+
+ for (const id of ['first', 'second', 'third']) {
+ reporter.enqueueHostOutcome(outcome(id))
+ await settle()
+ }
+ reporter.enqueueHostOutcome(outcome('third'))
+ reporter.enqueueHostOutcome(outcome('first'))
+ await settle()
+
+ // 'third' is still remembered (witness); 'first' was evicted and is sent again.
+ expect(submittedIds(fetchFn)).toEqual(['first', 'second', 'third', 'first'])
+ expect(await counted('deduplicated')).toBe(1)
+ })
+})
diff --git a/host-context-controller/src/administrativeOutcomeReporter.ts b/host-context-controller/src/administrativeOutcomeReporter.ts
index 8e1503c2b..f14e252d7 100644
--- a/host-context-controller/src/administrativeOutcomeReporter.ts
+++ b/host-context-controller/src/administrativeOutcomeReporter.ts
@@ -1,6 +1,7 @@
import { BoundedOffPathReporter } from './boundedOffPathReporter'
import { config } from './config'
import { administrativeOutcomeReporterTotal } from './metrics'
+import { throwForFailedSubmit } from './reporterHttpFailure'
import { signInternalControlJwt } from './utils/internalControlSigner'
export type AdministrativeHostOutcomeProjection = {
@@ -24,12 +25,14 @@ export type AdministrativeOutcomeReporterDependencies = {
retryLimit?: number
timeoutMs?: number
random?: () => number
+ dedupeCapacity?: number
}
const DEFAULT_CAPACITY = 64
const DEFAULT_RETRY_LIMIT = 2
const DEFAULT_TIMEOUT_MS = 1_000
const DEFAULT_STOP_TIMEOUT_MS = 1_500
+const DEFAULT_DEDUPE_CAPACITY = 1_024
export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcomeReporter {
private readonly queue: BoundedOffPathReporter<AdministrativeHostOutcomeProjection>
@@ -37,12 +40,31 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutco
private readonly baseUrl: string
private readonly signToken: () => string
private readonly fetchFn: typeof fetch
+ private readonly dedupeCapacity: number
+ // The reconciler re-observes the same outcome on every pass (3-4 fleet
+ // passes per 5 minutes) and its sourceEventId is deterministic, so each
+ // fact is sent once per process (#327). A key stays here while it is queued,
+ // in flight, accepted or rejected as terminal; only a drop removes it.
+ // Iteration order is least recently seen first, which is the eviction
+ // order. Past `dedupeCapacity` distinct keys per pass, the oldest are
+ // evicted and resent; control-api answers those resends as replays.
+ private readonly seen = new Set<string>()
constructor(deps: AdministrativeOutcomeReporterDependencies) {
const capacity = deps.capacity ?? DEFAULT_CAPACITY
if (!Number.isSafeInteger(capacity) || capacity < 1) {
throw new Error('administrative outcome reporter capacity must be a positive integer')
}
+ const dedupeCapacity = deps.dedupeCapacity ?? DEFAULT_DEDUPE_CAPACITY
+ // Up to `capacity` keys are buffered plus one in flight. A smaller set
+ // would evict a key that is still queued, and the next pass would queue
+ // it a second time.
+ if (!Number.isSafeInteger(dedupeCapacity) || dedupeCapacity <= capacity) {
+ throw new Error(
+ 'administrative outcome reporter dedupeCapacity must be an integer greater than capacity'
+ )
+ }
+ this.dedupeCapacity = dedupeCapacity
this.timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS
this.baseUrl = (deps.baseUrl ?? config.controlApiBaseUrl).replace(/\/$/, '')
this.signToken = deps.signToken ?? signInternalControlJwt
@@ -55,11 +77,30 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutco
submit: projection => this.submit(projection),
onEnqueued: () => administrativeOutcomeReporterTotal.inc({ result: 'enqueued' }),
onAccepted: () => administrativeOutcomeReporterTotal.inc({ result: 'accepted' }),
- onDrop: (_projection, reason) => administrativeOutcomeReporterTotal.inc({ result: reason }),
+ onTerminal: (_projection, result) => administrativeOutcomeReporterTotal.inc({ result }),
+ onDrop: (projection, reason) => {
+ // A dropped outcome was never delivered; forgetting it lets the next
+ // reconcile pass enqueue it again, so no outcome is lost.
+ this.seen.delete(projection.sourceEventId)
+ administrativeOutcomeReporterTotal.inc({ result: reason })
+ },
})
}
enqueueHostOutcome(projection: AdministrativeHostOutcomeProjection): void {
+ const { sourceEventId } = projection
+ if (this.seen.delete(sourceEventId)) {
+ // Re-inserted as most recently seen, so a key observed on every pass is
+ // not evicted behind keys observed once.
+ this.seen.add(sourceEventId)
+ administrativeOutcomeReporterTotal.inc({ result: 'deduplicated' })
+ return
+ }
+ if (this.seen.size >= this.dedupeCapacity) {
+ const oldest = this.seen.values().next().value
+ if (oldest !== undefined) this.seen.delete(oldest)
+ }
+ this.seen.add(sourceEventId)
this.queue.enqueue({ ...projection, hostRef: { ...projection.hostRef } })
}
@@ -97,9 +138,7 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutco
signal: controller.signal,
}
)
- if (!response.ok) {
- throw new Error(`administrative outcome submit failed with ${response.status}`)
- }
+ await throwForFailedSubmit(response, 'administrative outcome')
} finally {
clearTimeout(timeout)
}
diff --git a/host-context-controller/src/boundedOffPathReporter.test.ts b/host-context-controller/src/boundedOffPathReporter.test.ts
index 5b8d8a216..742568148 100644
--- a/host-context-controller/src/boundedOffPathReporter.test.ts
+++ b/host-context-controller/src/boundedOffPathReporter.test.ts
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'
import {
BoundedOffPathReporter,
type BoundedOffPathReporterOptions,
+ ReporterTerminalError,
} from './boundedOffPathReporter'
function createReporter(overrides: Partial<BoundedOffPathReporterOptions<string>> = {}) {
@@ -12,6 +13,7 @@ function createReporter(overrides: Partial<BoundedOffPathReporterOptions<string>
random: () => 0,
submit: vi.fn().mockResolvedValue(undefined),
onAccepted: vi.fn(),
+ onTerminal: vi.fn(),
onDrop: vi.fn(),
...overrides,
})
@@ -69,6 +71,25 @@ describe('BoundedOffPathReporter', () => {
expect(onDrop).toHaveBeenCalledWith('event-1', 'retry_exhausted')
})
+ it('settles a terminal rejection once, without retry or drop (#326)', async () => {
+ const submit = vi
+ .fn()
+ .mockRejectedValue(new ReporterTerminalError('conflict', 'submit rejected with 409'))
+ const onTerminal = vi.fn()
+ const onRetry = vi.fn()
+ const onDrop = vi.fn()
+ const reporter = createReporter({ retryLimit: 2, submit, onTerminal, onRetry, onDrop })
+
+ reporter.enqueue('event-1')
+ await new Promise(resolve => setTimeout(resolve, 150))
+
+ // Liveness: the submission ran and its rejection was classified.
+ expect(submit).toHaveBeenCalledOnce()
+ expect(onTerminal).toHaveBeenCalledExactlyOnceWith('event-1', 'conflict')
+ expect(onRetry).not.toHaveBeenCalled()
+ expect(onDrop).not.toHaveBeenCalled()
+ })
+
it('drains queued entries during stop and rejects later enqueue', async () => {
const submit = vi.fn().mockResolvedValue(undefined)
const onDrop = vi.fn()
diff --git a/host-context-controller/src/boundedOffPathReporter.ts b/host-context-controller/src/boundedOffPathReporter.ts
index 9c2a221b4..ec12962d1 100644
--- a/host-context-controller/src/boundedOffPathReporter.ts
+++ b/host-context-controller/src/boundedOffPathReporter.ts
@@ -6,6 +6,22 @@ export type BoundedReporterDropReason =
| 'retry_exhausted'
| 'shutdown_timeout'
+export type ReporterTerminalResult = 'conflict' | 'rejected'
+
+/**
+ * A submission the receiver rejected deterministically. Resending the same
+ * value yields the same answer, so the queue settles it without a retry (#326).
+ */
+export class ReporterTerminalError extends Error {
+ constructor(
+ readonly result: ReporterTerminalResult,
+ message: string
+ ) {
+ super(message)
+ this.name = 'ReporterTerminalError'
+ }
+}
+
type BufferedEntry<Value> = { value: Value; attempts: number }
export type BoundedOffPathReporterOptions<Value> = {
@@ -17,6 +33,7 @@ export type BoundedOffPathReporterOptions<Value> = {
onEnqueued?: (value: Value) => void
onAccepted: (value: Value) => void
onRetry?: (value: Value) => void
+ onTerminal: (value: Value, result: ReporterTerminalResult) => void
onDrop: (value: Value, reason: BoundedReporterDropReason) => void
}
@@ -115,7 +132,11 @@ export class BoundedOffPathReporter<Value> {
try {
await this.options.submit(entry.value)
this.options.onAccepted(entry.value)
- } catch {
+ } catch (error) {
+ if (error instanceof ReporterTerminalError) {
+ this.options.onTerminal(entry.value, error.result)
+ return
+ }
if (this.stopped || entry.attempts >= this.options.retryLimit) {
this.options.onDrop(
entry.value,
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 459e738ea..2f8242593 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -19,6 +19,7 @@ import { GFS_HOST_SCOPES } from './gfsHostPolicy'
import { makeExpectedHostGfsSubject } from './gfsHostSubject'
import type {
HccInfrastructureTelemetryPayload,
+ HostLookupReference,
InfrastructureTelemetryReporter,
} from './infrastructureTelemetryReporter'
import {
@@ -502,12 +503,8 @@ type RuntimeTokenProvision = {
scopeHash: string
}
-type GfsTokenLifecycleEvidence = {
- gfs_subject: string
- gfs_outcome: 'minted' | 'rotated' | 'reused' | 'failed'
- gfs_old_host_uid?: string
- gfs_new_host_uid?: string
-}
+// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).
+type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'
type DeploymentMutationState = {
lifecycle: EffectiveHostLifecycle
@@ -558,7 +555,7 @@ export class HostReconciler {
) => CommunicationChannelCRD[]
private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporter
private readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter
- private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleEvidence>()
+ private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleOutcome>()
// B2: whether the CC cache initial-list has completed. Defaults to false
// (safe: preserves existing Deployment replicas until wired by McpServerWatcher).
private ccCacheSyncedFn: () => boolean = () => false
@@ -652,11 +649,7 @@ export class HostReconciler {
this.infrastructureTelemetryReporter?.enqueueHealthTransition({
sourceEventId: `hcc-health-transition:${this.newTelemetryOccurrenceId()}`,
occurredAt,
- hostLookupReference: {
- name: host.name,
- namespace: host.namespace,
- ...(host.generation !== undefined ? { generation: host.generation } : {}),
- },
+ hostLookupReference: this.hostLookupReference(host),
payload: { transition: `lifecycle:${lifecycle.state}`, state: lifecycle.state },
})
},
@@ -1850,11 +1843,7 @@ export class HostReconciler {
resourceName: name,
refreshExpInHours,
})
- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {
- gfs_subject: expectedGfsSubject,
- gfs_outcome: 'reused',
- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),
- })
+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'reused')
const selectedRevision =
rolloutPending || HostReconciler.shouldRollForRuntimeSecret(deployment, false)
? existingRevision
@@ -1934,11 +1923,7 @@ export class HostReconciler {
await observeCreate('Secret', () =>
this.coreApi.createNamespacedSecret({ namespace: host.namespace, body })
)
- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {
- gfs_subject: expectedGfsSubject,
- gfs_outcome: 'minted',
- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),
- })
+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'minted')
log.info('created mcp-host-runtime-token Secret', {
host: host.name,
namespace: host.namespace,
@@ -1960,13 +1945,7 @@ export class HostReconciler {
namespace: host.namespace,
body: replaceBody,
})
- const oldHostUid = existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION]
- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {
- gfs_subject: expectedGfsSubject,
- gfs_outcome: 'rotated',
- ...(oldHostUid ? { gfs_old_host_uid: oldHostUid } : {}),
- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),
- })
+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'rotated')
log.info('rotated mcp-host-runtime-token Secret', {
host: host.name,
namespace: host.namespace,
@@ -1982,13 +1961,8 @@ export class HostReconciler {
}
} catch (err) {
lastErr = err
- const subject = makeExpectedHostGfsSubject(host.namespace, host.name)
- if (subject) {
- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {
- gfs_subject: subject,
- gfs_outcome: 'failed',
- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),
- })
+ if (makeExpectedHostGfsSubject(host.namespace, host.name)) {
+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'failed')
}
if (err instanceof Error && err.name === 'GfsHostTokenValidationError') break
const delayMs = 1000 * Math.pow(2, attempt - 1)
@@ -2154,15 +2128,12 @@ export class HostReconciler {
}
}
- private hostLookupReference(host: HostCRD): {
- name: string
- namespace: string
- generation?: number
- } {
+ private hostLookupReference(host: HostCRD): HostLookupReference {
return {
name: host.name,
namespace: host.namespace,
...(host.generation !== undefined ? { generation: host.generation } : {}),
+ ...(host.uid !== undefined ? { uid: host.uid } : {}),
}
}
@@ -2207,16 +2178,38 @@ export class HostReconciler {
private enqueueReconcileOutcome(host: HostCRD): void {
const status = this.getStatus(host.name)
const succeeded = status.deployed && status.ready
- this.enqueueHostTelemetry(host, 'reconcile_outcome', succeeded ? 'ready' : 'not_ready', {
+ this.enqueueReconcileTelemetry(host, succeeded ? 'ready' : 'not_ready', {
status: succeeded ? 'succeeded' : 'failed',
phase: status.deployed ? 'deployed' : 'not_deployed',
state: status.ready ? 'ready' : 'not_ready',
- ...(this.gfsTokenLifecycleEvidence.get(HostReconciler.gfsLifecycleEvidenceKey(host)) ?? {}),
})
- this.gfsTokenLifecycleEvidence.delete(HostReconciler.gfsLifecycleEvidenceKey(host))
if (succeeded) this.enqueueAdministrativeOutcome(host, 'succeeded', 'reconciled')
}
+ /**
+ * A pass that threw may not have written any status yet, so the status in
+ * memory can still be the previous pass's. Report the failure itself, not
+ * that status, and never an administrative `succeeded` (#327).
+ */
+ private enqueueReconcileExceptionOutcome(host: HostCRD): void {
+ this.enqueueReconcileTelemetry(host, 'reconcile_exception', { status: 'failed' })
+ }
+
+ private enqueueReconcileTelemetry(
+ host: HostCRD,
+ reasonCode: string,
+ payload: HccInfrastructureTelemetryPayload
+ ): void {
+ const evidenceKey = HostReconciler.gfsLifecycleEvidenceKey(host)
+ const gfsOutcome = this.gfsTokenLifecycleEvidence.get(evidenceKey)
+ this.gfsTokenLifecycleEvidence.delete(evidenceKey)
+ this.enqueueHostTelemetry(host, 'reconcile_outcome', reasonCode, {
+ ...payload,
+ // control-api allowlists `transition`, not gfs_* keys (#328).
+ ...(gfsOutcome ? { transition: `gfs_token:${gfsOutcome}` } : {}),
+ })
+ }
+
private enqueueAdministrativeOutcome(
host: HostCRD,
outcome: 'succeeded' | 'failed',
@@ -4435,11 +4428,19 @@ export class HostReconciler {
// or an administrative 'failed' outcome. Callers already treat the
// rethrow as a retire (reconcileDelete → 'superseded', watch callers
// only log the error), so the rethrow is preserved.
+ // This exit emits no outcome, so drop the GFS evidence this pass
+ // recorded. Keyed by namespace/name, a leftover would be reported by
+ // the next pass, or by a recreated Host with the same name. The
+ // stateless suspension path also calls reconcileCore without emitting
+ // an outcome and does not clear the entry (#696).
+ this.gfsTokenLifecycleEvidence.delete(
+ HostReconciler.gfsLifecycleEvidenceKey(admittedHost)
+ )
this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)
throw error
}
this.enqueueControllerError(admittedHost, 'reconcile_exception', error)
- this.enqueueReconcileOutcome(admittedHost)
+ this.enqueueReconcileExceptionOutcome(admittedHost)
this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)
throw error
} finally {
diff --git a/host-context-controller/src/infrastructureTelemetryReporter.test.ts b/host-context-controller/src/infrastructureTelemetryReporter.test.ts
index 8cf633e10..27ea9c311 100644
--- a/host-context-controller/src/infrastructureTelemetryReporter.test.ts
+++ b/host-context-controller/src/infrastructureTelemetryReporter.test.ts
@@ -4,7 +4,11 @@ import {
createInfrastructureTelemetryReporter,
hccReconcileOutcomeSourceId,
} from './infrastructureTelemetryReporter'
-import { infrastructureTelemetryGapsTotal, infrastructureTelemetryRetriesTotal } from './metrics'
+import {
+ infrastructureTelemetryFlushesTotal,
+ infrastructureTelemetryGapsTotal,
+ infrastructureTelemetryRetriesTotal,
+} from './metrics'
const projection = {
sourceEventId: 'hcc-health-transition:mcp-host:chatllm:7:active:1',
@@ -134,7 +138,7 @@ describe('BoundedInfrastructureTelemetryReporter', () => {
it('keeps reconcile identity stable for equivalent state and changes it with the outcome', () => {
expect(hccReconcileOutcomeSourceId(reconcileProjection)).toMatch(
- /^hcc-reconcile-outcome-v2:[0-9a-f]{64}$/
+ /^hcc-reconcile-outcome-v3:[0-9a-f]{64}$/
)
expect(hccReconcileOutcomeSourceId(reconcileProjection)).toBe(
hccReconcileOutcomeSourceId({ ...reconcileProjection })
@@ -151,6 +155,108 @@ describe('BoundedInfrastructureTelemetryReporter', () => {
)
})
+ it('gives a recreated Host object (same name and generation, new uid) new identities (#691)', () => {
+ const withUid = (uid: string) => ({
+ ...reconcileProjection,
+ hostLookupReference: { ...reconcileProjection.hostLookupReference, uid },
+ })
+
+ // Liveness: the identity is deterministic for an identical tuple.
+ expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).toBe(
+ hccReconcileOutcomeSourceId(withUid('uid-new'))
+ )
+ expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).not.toBe(
+ hccReconcileOutcomeSourceId(withUid('uid-old'))
+ )
+ })
+
+ it('settles a 400 unsafe_tracing_input once and keeps retrying a 500 (#326, #328)', async () => {
+ infrastructureTelemetryFlushesTotal.reset()
+ infrastructureTelemetryGapsTotal.reset()
+ const response = (status: number, body: unknown) =>
+ ({ ok: false, status, json: async () => body }) as unknown as Response
+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {
+ const event = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })
+ .events[0]!
+ return event.sourceEventId === 'rejected-event'
+ ? response(400, { code: 'unsafe_tracing_input', error: 'unsafe' })
+ : response(500, { error: 'Internal server error' })
+ }) as unknown as typeof fetch
+ const reporter = new BoundedInfrastructureTelemetryReporter({
+ baseUrl: 'http://control-api.test:8090',
+ signToken: () => 'signed-request',
+ fetchFn,
+ random: () => 0,
+ })
+
+ reporter.enqueue({
+ ...projection,
+ telemetryType: 'lifecycle_transition',
+ sourceEventId: 'rejected-event',
+ })
+ reporter.enqueue({
+ ...projection,
+ telemetryType: 'lifecycle_transition',
+ sourceEventId: 'transient-event',
+ })
+ await new Promise(resolve => setTimeout(resolve, 150))
+
+ const ids = vi
+ .mocked(fetchFn)
+ .mock.calls.map(
+ call =>
+ (JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> })
+ .events[0]!.sourceEventId
+ )
+ // Liveness: the 500 on the other event is retried twice.
+ expect(ids.filter(id => id === 'transient-event')).toHaveLength(3)
+ expect(ids.filter(id => id === 'rejected-event')).toHaveLength(1)
+ const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
+ expect(flushes.find(value => value.labels.result === 'rejected')?.value).toBe(1)
+ expect(flushes.find(value => value.labels.result === 'exhausted')?.value).toBe(1)
+ // Both events were lost: one rejected, one after its retries.
+ const gaps = (await infrastructureTelemetryGapsTotal.get()).values.map(value => value.labels)
+ expect(gaps).toEqual(
+ expect.arrayContaining([
+ { telemetry_type: 'lifecycle_transition', reason: 'rejected' },
+ { telemetry_type: 'lifecycle_transition', reason: 'retry_exhausted' },
+ ])
+ )
+ expect(gaps).toHaveLength(2)
+ })
+
+ it('does not count a 409 idempotency conflict as an evidence gap', async () => {
+ infrastructureTelemetryFlushesTotal.reset()
+ infrastructureTelemetryGapsTotal.reset()
+ const fetchFn = vi.fn(
+ async () =>
+ ({
+ ok: false,
+ status: 409,
+ json: async () => ({ code: 'tracing_idempotency_conflict' }),
+ }) as unknown as Response
+ ) as unknown as typeof fetch
+ const reporter = new BoundedInfrastructureTelemetryReporter({
+ baseUrl: 'http://control-api.test:8090',
+ signToken: () => 'signed-request',
+ fetchFn,
+ random: () => 0,
+ })
+
+ reporter.enqueue({
+ ...projection,
+ telemetryType: 'lifecycle_transition',
+ sourceEventId: 'conflicting-event',
+ })
+ await new Promise(resolve => setTimeout(resolve, 150))
+
+ // Liveness: the conflict was submitted once and settled.
+ expect(fetchFn).toHaveBeenCalledOnce()
+ const flushes = (await infrastructureTelemetryFlushesTotal.get()).values
+ expect(flushes.find(value => value.labels.result === 'conflict')?.value).toBe(1)
+ expect((await infrastructureTelemetryGapsTotal.get()).values).toEqual([])
+ })
+
it('isolates a blackholed submission and continues flushing later telemetry', async () => {
infrastructureTelemetryGapsTotal.reset()
const fetchFn = vi
diff --git a/host-context-controller/src/infrastructureTelemetryReporter.ts b/host-context-controller/src/infrastructureTelemetryReporter.ts
index e7cf0441d..47c687f88 100644
--- a/host-context-controller/src/infrastructureTelemetryReporter.ts
+++ b/host-context-controller/src/infrastructureTelemetryReporter.ts
@@ -8,15 +8,28 @@ import {
infrastructureTelemetryGapsTotal,
infrastructureTelemetryRetriesTotal,
} from './metrics'
+import { throwForFailedSubmit } from './reporterHttpFailure'
import { signInternalControlJwt } from './utils/internalControlSigner'
export type HccHealthTransitionProjection = {
sourceEventId: string
occurredAt: string
- hostLookupReference: { name: string; namespace: string; generation?: number }
+ hostLookupReference: HostLookupReference
payload: { transition: string; state: string }
}
+/**
+ * `uid` pins the event to one Host object. A Host deleted and recreated with
+ * the same name restarts at generation 1; without the uid its events would
+ * reuse the previous object's identities (#691).
+ */
+export type HostLookupReference = {
+ name: string
+ namespace: string
+ generation?: number
+ uid?: string
+}
+
export type HccInfrastructureTelemetryType =
| 'health_transition'
| 'lifecycle_transition'
@@ -35,15 +48,11 @@ export type HccInfrastructureTelemetryPayload = {
detail_ref?: string
attempt?: number
count?: number
- gfs_subject?: string
- gfs_outcome?: 'minted' | 'rotated' | 'reused' | 'failed'
- gfs_old_host_uid?: string
- gfs_new_host_uid?: string
}
type HccInfrastructureTelemetryProjectionBase = {
occurredAt: string
- hostLookupReference: { name: string; namespace: string; generation?: number }
+ hostLookupReference: HostLookupReference
payload?: HccInfrastructureTelemetryPayload
}
@@ -67,6 +76,7 @@ export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProje
projection.hostLookupReference.namespace,
projection.hostLookupReference.name,
projection.hostLookupReference.generation ?? 0,
+ projection.hostLookupReference.uid ?? null,
payload.reason_code ?? null,
payload.error_class ?? null,
payload.phase ?? null,
@@ -78,12 +88,10 @@ export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProje
payload.detail_ref ?? null,
payload.attempt ?? null,
payload.count ?? null,
- payload.gfs_subject ?? null,
- payload.gfs_outcome ?? null,
- payload.gfs_old_host_uid ?? null,
- payload.gfs_new_host_uid ?? null,
]
- return `hcc-reconcile-outcome-v2:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`
+ // v3: the tuple gained the Host uid (#691) and lost the gfs_* fields, which
+ // control-api never accepted (#328). v2 identities are not produced again.
+ return `hcc-reconcile-outcome-v3:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`
}
export interface InfrastructureTelemetryReporter {
@@ -143,6 +151,17 @@ export class BoundedInfrastructureTelemetryReporter implements InfrastructureTel
onEnqueued: projection =>
infrastructureTelemetryEnqueuedTotal.inc({ telemetry_type: projection.telemetryType }),
onAccepted: () => infrastructureTelemetryFlushesTotal.inc({ result: 'accepted' }),
+ onTerminal: (projection, result) => {
+ infrastructureTelemetryFlushesTotal.inc({ result })
+ // A conflict means a row already exists for the key. A rejected event
+ // is never stored, so it is a gap in the evidence.
+ if (result === 'rejected') {
+ infrastructureTelemetryGapsTotal.inc({
+ telemetry_type: projection.telemetryType,
+ reason: result,
+ })
+ }
+ },
onRetry: projection =>
infrastructureTelemetryRetriesTotal.inc({ telemetry_type: projection.telemetryType }),
onDrop: (projection, reason) => this.recordDrop(projection.telemetryType, reason),
@@ -210,7 +229,7 @@ export class BoundedInfrastructureTelemetryReporter implements InfrastructureTel
signal: controller.signal,
}
)
- if (!response.ok) throw new Error(`telemetry submit failed with ${response.status}`)
+ await throwForFailedSubmit(response, 'telemetry')
} finally {
clearTimeout(timeout)
}
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 135108158..4be4dfa82 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -239,7 +239,7 @@ export const infrastructureTelemetryGapsTotal = counter({
export const administrativeOutcomeReporterTotal = counter({
name: 'clerum_hcc_administrative_outcome_reporter_total',
- help: 'Administrative outcome reporter enqueue, flush, and drop outcomes.',
+ help: 'Administrative outcome reporter results: enqueued, deduplicated, accepted, conflict, rejected, or a drop reason.',
labelNames: ['result'] as const,
})
diff --git a/host-context-controller/src/reporterHttpFailure.test.ts b/host-context-controller/src/reporterHttpFailure.test.ts
new file mode 100644
index 000000000..e2af52fc7
--- /dev/null
+++ b/host-context-controller/src/reporterHttpFailure.test.ts
@@ -0,0 +1,113 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { ReporterTerminalError } from './boundedOffPathReporter'
+import { HostContextLogger } from './logger'
+import { throwForFailedSubmit } from './reporterHttpFailure'
+
+function response(status: number, readBody: () => Promise<unknown>) {
+ const json = vi.fn(readBody)
+ return { response: { ok: status < 300, status, json } as unknown as Response, json }
+}
+
+function named(name: string): Error {
+ const error = new Error(`${name} while reading the body`)
+ error.name = name
+ return error
+}
+
+async function failure(status: number, readBody: () => Promise<unknown>) {
+ const { response: res } = response(status, readBody)
+ return throwForFailedSubmit(res, 'test event').then(
+ () => undefined,
+ (error: unknown) => error
+ )
+}
+
+afterEach(() => vi.restoreAllMocks())
+
+describe('throwForFailedSubmit', () => {
+ it('returns for a 2xx without reading the body', async () => {
+ const { response: res, json } = response(202, async () => ({}))
+
+ await expect(throwForFailedSubmit(res, 'test event')).resolves.toBeUndefined()
+ expect(json).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ [409, 'tracing_idempotency_conflict', 'conflict'],
+ [400, 'unsafe_tracing_input', 'rejected'],
+ [400, 'invalid_tracing_input', 'rejected'],
+ ] as const)('treats %i %s as terminal %s', async (status, code, result) => {
+ const error = await failure(status, async () => ({ code, error: 'ignored' }))
+
+ expect(error).toBeInstanceOf(ReporterTerminalError)
+ expect(error).toMatchObject({ result })
+ })
+
+ it.each([
+ [400, 'tracing_idempotency_conflict'],
+ [409, 'unsafe_tracing_input'],
+ [409, 'invalid_tracing_input'],
+ [403, 'tracing_idempotency_conflict'],
+ ] as const)(
+ 'keeps %i with %s retryable: the status is part of the pair',
+ async (status, code) => {
+ const error = await failure(status, async () => ({ code }))
+
+ expect(error).toBeInstanceOf(Error)
+ expect(error).not.toBeInstanceOf(ReporterTerminalError)
+ expect((error as Error).message).toBe(`test event submit failed with ${status}`)
+ }
+ )
+
+ it.each([
+ ['an array', [{ code: 'tracing_idempotency_conflict' }]],
+ ['null', null],
+ ['a numeric code', { code: 409 }],
+ ['a string', 'tracing_idempotency_conflict'],
+ ])('keeps a 409 whose body is %s retryable', async (_label, body) => {
+ const error = await failure(409, async () => body)
+
+ expect(error).toBeInstanceOf(Error)
+ expect(error).not.toBeInstanceOf(ReporterTerminalError)
+ })
+
+ it('does not read the body of a status that has no terminal pair', async () => {
+ const { response: res, json } = response(500, async () => ({ code: 'unsafe_tracing_input' }))
+
+ await expect(throwForFailedSubmit(res, 'test event')).rejects.toThrow(
+ 'test event submit failed with 500'
+ )
+ expect(json).not.toHaveBeenCalled()
+ })
+
+ it('keeps the body out of the error message', async () => {
+ const error = await failure(409, async () => ({
+ code: 'tracing_idempotency_conflict',
+ error: 'body-marker',
+ }))
+
+ // Liveness: the body was read, since the code made it terminal.
+ expect(error).toBeInstanceOf(ReporterTerminalError)
+ expect((error as Error).message).not.toContain('body-marker')
+ })
+
+ it.each([
+ ['SyntaxError', 'not_json'],
+ ['AbortError', 'aborted'],
+ ['TypeError', 'read_failed'],
+ ])('logs a %s during the body read as %s and stays retryable', async (name, reason) => {
+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => {})
+
+ const error = await failure(409, async () => {
+ throw named(name)
+ })
+
+ expect(error).not.toBeInstanceOf(ReporterTerminalError)
+ expect(warn).toHaveBeenCalledOnce()
+ expect(warn).toHaveBeenCalledWith('reporter could not read the response code', {
+ label: 'test event',
+ status: 409,
+ reason,
+ })
+ })
+})
diff --git a/host-context-controller/src/reporterHttpFailure.ts b/host-context-controller/src/reporterHttpFailure.ts
new file mode 100644
index 000000000..52062ae02
--- /dev/null
+++ b/host-context-controller/src/reporterHttpFailure.ts
@@ -0,0 +1,65 @@
+import { ReporterTerminalError, type ReporterTerminalResult } from './boundedOffPathReporter'
+import { hccLogger } from './logger'
+
+const log = hccLogger.child({ module: 'reporter-http-failure' })
+
+/**
+ * control-api answers these (status, code) pairs deterministically: resending
+ * the same event gets the same answer. `invalid_tracing_input` comes only from
+ * request shape validation, so it is as final as `unsafe_tracing_input`. Every other failure (403 binding not yet
+ * visible, 5xx, a 4xx without a code, a body that is not JSON) stays retryable.
+ */
+const TERMINAL_RESPONSES: ReadonlyArray<{
+ status: number
+ code: string
+ result: ReporterTerminalResult
+}> = [
+ { status: 409, code: 'tracing_idempotency_conflict', result: 'conflict' },
+ { status: 400, code: 'unsafe_tracing_input', result: 'rejected' },
+ { status: 400, code: 'invalid_tracing_input', result: 'rejected' },
+]
+
+function unreadableBodyReason(error: unknown): 'not_json' | 'aborted' | 'read_failed' {
+ if (error instanceof Error && error.name === 'SyntaxError') return 'not_json'
+ if (error instanceof Error && error.name === 'AbortError') return 'aborted'
+ return 'read_failed'
+}
+
+async function responseCode(response: Response, label: string): Promise<string | undefined> {
+ let body: unknown
+ try {
+ body = await response.json()
+ } catch (error) {
+ // No code can be read, so the caller treats the response as retryable.
+ // Only the status and the failure kind are logged, never the body.
+ log.warn('reporter could not read the response code', {
+ label,
+ status: response.status,
+ reason: unreadableBodyReason(error),
+ })
+ return undefined
+ }
+ if (typeof body !== 'object' || body === null) return undefined
+ const code = (body as { code?: unknown }).code
+ return typeof code === 'string' ? code : undefined
+}
+
+/**
+ * Throws for a non-2xx submit response. Only the `code` field of the body is
+ * read; the body is never logged or attached to the error.
+ */
+export async function throwForFailedSubmit(response: Response, label: string): Promise<void> {
+ if (response.ok) return
+ const candidates = TERMINAL_RESPONSES.filter(entry => entry.status === response.status)
+ if (candidates.length > 0) {
+ const code = await responseCode(response, label)
+ const terminal = candidates.find(entry => entry.code === code)
+ if (terminal) {
+ throw new ReporterTerminalError(
+ terminal.result,
+ `${label} submit rejected with ${response.status} ${terminal.code}`
+ )
+ }
+ }
+ throw new Error(`${label} submit failed with ${response.status}`)
+}
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 8e5f85fc6..b3e79f1ad 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1493,6 +1493,26 @@ describe('HostReconciler stateless lifecycle — status write idempotence', () =
).toHaveLength(1)
})
+ it('binds the committed health transition to the Host uid (#691)', async () => {
+ const infrastructureTelemetryReporter = createTelemetryReporterMock()
+ const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
+ await reconciler.reconcile({ ...makeStatelessHost(), generation: 3 })
+
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledTimes(1)
+ expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledWith(
+ expect.objectContaining({
+ hostLookupReference: {
+ name: 'stateless-host',
+ namespace: 'mcp-host',
+ generation: 3,
+ uid: 'stateless-host-uid',
+ },
+ payload: { transition: 'lifecycle:suspended', state: 'suspended' },
+ })
+ )
+ })
+
it('skips the write when the observed status already matches', async () => {
const infrastructureTelemetryReporter = createTelemetryReporterMock()
const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index 062ad5e12..7100fb1b9 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -1,6 +1,9 @@
-import { describe, expect, it, vi } from 'vitest'
+import { describe, expect, it, onTestFinished, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
-import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeReporter'
+import {
+ type AdministrativeOutcomeReporter,
+ BoundedAdministrativeOutcomeReporter,
+} from '../src/administrativeOutcomeReporter'
import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,
@@ -271,16 +274,24 @@ describe('HostReconciler', () => {
'clerum.io/gfs-token-host-generation': '1',
'clerum.io/runtime-token-rollout-required': 'true',
})
- expect(reporter.enqueue).toHaveBeenCalledWith(
- expect.objectContaining({
- telemetryType: 'reconcile_outcome',
- payload: expect.objectContaining({
- gfs_subject: 'host:1st:mcp-host/alpha-host',
- gfs_old_host_uid: 'old-host-uid',
- gfs_new_host_uid: 'new-host-uid',
- gfs_outcome: 'rotated',
- }),
- })
+ // GFS evidence travels as the allowlisted `transition` (#328); the recreated
+ // object is identified by the uid in the lookup reference (#691).
+ const recreationOutcome = vi
+ .mocked(reporter.enqueue)
+ .mock.calls.map(([projection]) => projection)
+ .filter(projection => projection.telemetryType === 'reconcile_outcome')
+ .at(-1)
+ expect(recreationOutcome).toMatchObject({
+ hostLookupReference: {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ generation: 1,
+ uid: 'new-host-uid',
+ },
+ payload: { transition: 'gfs_token:rotated' },
+ })
+ expect(Object.keys(recreationOutcome!.payload!).filter(key => key.startsWith('gfs_'))).toEqual(
+ []
)
expect(
appsApi.replaceNamespacedDeployment.mock.calls.every(
@@ -350,7 +361,12 @@ describe('HostReconciler', () => {
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'reconcile_outcome',
- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 7 },
+ hostLookupReference: {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ generation: 7,
+ uid: 'host-uid-1',
+ },
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'ready',
@@ -389,6 +405,210 @@ describe('HostReconciler', () => {
)
})
+ it('sends an administrative outcome once across repeated reconcile passes (#327)', async () => {
+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch
+ const administrativeOutcomeReporter = new BoundedAdministrativeOutcomeReporter({
+ baseUrl: 'http://control-api.test:8090',
+ signToken: () => 'signed',
+ fetchFn,
+ random: () => 0,
+ })
+ onTestFinished(() => administrativeOutcomeReporter.stop())
+ const telemetry = createTelemetryReporterMock()
+ const { reconciler } = createReconciler({
+ administrativeOutcomeReporter,
+ infrastructureTelemetryReporter: telemetry,
+ })
+ const host = makeHost({
+ generation: 7,
+ annotations: { 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111' },
+ })
+
+ for (let pass = 0; pass < 3; pass += 1) {
+ await reconciler.reconcile(host)
+ await new Promise(resolve => setTimeout(resolve, 0))
+ }
+
+ // Liveness: every pass re-observed the outcome.
+ expect(
+ vi
+ .mocked(telemetry.enqueue)
+ .mock.calls.filter(([projection]) => projection.telemetryType === 'reconcile_outcome')
+ ).toHaveLength(3)
+ expect(fetchFn).toHaveBeenCalledOnce()
+ })
+
+ it('does not turn the previous pass status into a succeeded outcome when a pass throws (#327)', async () => {
+ const administrativeOutcomeReporter = {
+ enqueueHostOutcome: vi.fn(),
+ stop: vi.fn(async () => undefined),
+ }
+ const telemetry = createTelemetryReporterMock()
+ const { reconciler, rbacApi } = createReconciler({
+ administrativeOutcomeReporter,
+ infrastructureTelemetryReporter: telemetry,
+ })
+ const annotations = {
+ 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',
+ }
+ await reconciler.reconcile(makeHost({ generation: 7, annotations }))
+ administrativeOutcomeReporter.enqueueHostOutcome.mockClear()
+ vi.mocked(telemetry.enqueue).mockClear()
+ // Fails before this pass writes any status, so the status read afterwards
+ // is still the previous pass's deployed/ready.
+ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
+
+ await expect(reconciler.reconcile(makeHost({ generation: 8, annotations }))).rejects.toEqual({
+ code: 500,
+ })
+
+ // Liveness: the failed pass reported its error and its reconcile outcome
+ // while the previous pass's ready status was still in place.
+ expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
+ const types = vi
+ .mocked(telemetry.enqueue)
+ .mock.calls.map(([projection]) => projection.telemetryType)
+ expect(types).toContain('controller_error')
+ // The telemetry reports the exception, not the stale ready status.
+ const failedOutcome = vi
+ .mocked(telemetry.enqueue)
+ .mock.calls.map(([projection]) => projection)
+ .find(projection => projection.telemetryType === 'reconcile_outcome')
+ expect(failedOutcome?.payload).toEqual({
+ resource_class: 'Host',
+ reason_code: 'reconcile_exception',
+ status: 'failed',
+ })
+ expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(
+ expect.objectContaining({
+ outcome: 'failed',
+ hostRef: expect.objectContaining({ generation: 8 }),
+ })
+ )
+ expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalledWith(
+ expect.objectContaining({ outcome: 'succeeded' })
+ )
+ })
+
+ describe('GFS token evidence on reconcile_outcome (#328)', () => {
+ function reconcileOutcomes(reporter: InfrastructureTelemetryReporter) {
+ return vi
+ .mocked(reporter.enqueue)
+ .mock.calls.map(([projection]) => projection)
+ .filter(projection => projection.telemetryType === 'reconcile_outcome')
+ }
+
+ it('reports minted once and not on the next pass', async () => {
+ const reporter = createTelemetryReporterMock()
+ const { reconciler, rbacApi, coreApi } = createReconciler({
+ infrastructureTelemetryReporter: reporter,
+ })
+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
+ coreApi.readNamespacedSecret.mockImplementation((request: { name?: string } = {}) =>
+ request.name === 'host-alpha-host-mcp-host-runtime-tokens'
+ ? Promise.reject({ code: 404 })
+ : readSecret(request)
+ )
+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
+
+ await reconciler.reconcile(makeHost({ generation: 3 }))
+ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
+ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toEqual({
+ code: 500,
+ })
+
+ // Liveness: both passes emitted an outcome; only the first reached the mint.
+ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
+ const [first, second] = reconcileOutcomes(reporter)
+ expect(first?.payload).toMatchObject({ transition: 'gfs_token:minted' })
+ expect(second?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
+ expect(second?.payload).not.toHaveProperty('transition')
+ })
+
+ it('reports reused when the runtime Secret is kept', async () => {
+ const reporter = createTelemetryReporterMock()
+ const { reconciler, coreApi } = createReconciler({
+ infrastructureTelemetryReporter: reporter,
+ })
+ // Reuse needs a refresh token whose `exp` can be read.
+ const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url')
+ const refreshToken = [
+ encode({ alg: 'none' }),
+ encode({ exp: Math.floor(Date.now() / 1000) + 30 * 86_400 }),
+ 'unsigned',
+ ].join('.')
+ const issue = vi.mocked(issueMcpHostRuntimeTokens).getMockImplementation()!
+ vi.mocked(issueMcpHostRuntimeTokens).mockImplementationOnce(async (...args) => ({
+ ...(await issue(...args)),
+ refreshToken,
+ }))
+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
+
+ await reconciler.reconcile(makeHost({ generation: 3 }))
+ const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(
+ ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'
+ )?.[0].body as k8s.V1Secret
+ coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {
+ if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
+ return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
+ })
+ await reconciler.reconcile(makeHost({ generation: 3 }))
+
+ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
+ expect(reconcileOutcomes(reporter).map(outcome => outcome.payload?.transition)).toEqual([
+ 'gfs_token:rotated',
+ 'gfs_token:reused',
+ ])
+ })
+
+ it('reports failed on the exception outcome when the mint is rejected', async () => {
+ const reporter = createTelemetryReporterMock()
+ const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })
+ const validationError = new Error('gfs host token subject mismatch')
+ validationError.name = 'GfsHostTokenValidationError'
+ vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)
+
+ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toThrow(
+ /subject mismatch/
+ )
+
+ expect(reconcileOutcomes(reporter).map(outcome => outcome.payload)).toEqual([
+ {
+ resource_class: 'Host',
+ reason_code: 'reconcile_exception',
+ status: 'failed',
+ transition: 'gfs_token:failed',
+ },
+ ])
+ })
+
+ it('drops the evidence of a superseded pass', async () => {
+ const reporter = createTelemetryReporterMock()
+ const { reconciler, appsApi, rbacApi } = createReconciler({
+ infrastructureTelemetryReporter: reporter,
+ })
+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length
+ const superseded = new Error('Host spec changed while the Deployment write was in flight')
+ superseded.name = 'HostMutationSpecRevisionChangedError'
+ appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(superseded)
+
+ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toBe(superseded)
+ // Liveness: the superseded pass minted, and emitted no outcome.
+ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)
+ expect(reconcileOutcomes(reporter)).toHaveLength(0)
+
+ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })
+ await expect(reconciler.reconcile(makeHost({ generation: 4 }))).rejects.toEqual({
+ code: 500,
+ })
+
+ const outcomes = reconcileOutcomes(reporter)
+ expect(outcomes).toHaveLength(1)
+ expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })
+ expect(outcomes[0]?.payload).not.toHaveProperty('transition')
+ })
+ })
+
it('emits Host-backed controller_error and failed reconcile_outcome on reconcile failure', async () => {
const reporter = createTelemetryReporterMock()
const { reconciler, coreApi } = createReconciler({ infrastructureTelemetryReporter: reporter })
@@ -411,7 +631,12 @@ describe('HostReconciler', () => {
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'controller_error',
- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 8 },
+ hostLookupReference: {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ generation: 8,
+ uid: 'host-uid-1',
+ },
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'SecretNotFound',
@@ -423,7 +648,12 @@ describe('HostReconciler', () => {
expect(reporter.enqueue).toHaveBeenCalledWith(
expect.objectContaining({
telemetryType: 'reconcile_outcome',
- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 8 },
+ hostLookupReference: {
+ name: 'alpha-host',
+ namespace: 'mcp-host',
+ generation: 8,
+ uid: 'host-uid-1',
+ },
payload: expect.objectContaining({
resource_class: 'Host',
reason_code: 'not_ready',