Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc,control-api): stop tracing idempotency 409s for admin outcomes and recreated Hosts (#692)
ALFREDO J LOPEZ H (alfredolopez80.eth)ccd92ec04d96
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 9837b354e..5d2665b68 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.586",+ "version": "0.1.587","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.586",+ "version": "0.1.587","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",diff --git a/control-api/package.json b/control-api/package.jsonindex ddcea757c..9ac84821c 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.586",+ "version": "0.1.587","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/http/errorHandler.ts b/control-api/src/http/errorHandler.tsindex 07581b4b2..25f61b94a 100644--- a/control-api/src/http/errorHandler.ts+++ b/control-api/src/http/errorHandler.ts@@ -12,6 +12,16 @@ export const FORWARDABLE_INTEGRATION_CODES = new Set(['registry_integration_error',])+// Stable machine-readable codes added to a forwarded 4xx body. Internal+// reporters (HCC) use them to tell a deterministic rejection, which must not be+// retried, from a transient one. Only codes set by our own error constructors+// belong here; every other 4xx keeps the `{ error, correlationId }` shape.+export const FORWARDABLE_CLIENT_ERROR_CODES = new Set([+ 'tracing_idempotency_conflict',+ 'unsafe_tracing_input',+ 'invalid_tracing_input',+])+/*** Extract an HTTP status from an error the same way `extractK8sStatus`* (resourceServiceHelpers) does, so a raw @kubernetes/client-node error — which@@ -168,17 +178,24 @@ export function clerumErrorHandler(const isClientError = errStatus !== undefined && errStatus >= 400 && errStatus < 500if (isClientError) {+ const clientCode = (err as { code?: unknown }).code+ const forwardedCode =+ typeof clientCode === 'string' && FORWARDABLE_CLIENT_ERROR_CODES.has(clientCode)+ ? clientCode+ : undefinedlog.warn({event: 'forwarded_client_error',correlationId,status: errStatus,+ ...(forwardedCode ? { code: forwardedCode } : {}),err: safeErrorLogMessage(err),},'forwarded client error from upstream')res.status(errStatus as number).json({error: clientErrorMessage(err, errStatus as number),+ ...(forwardedCode ? { code: forwardedCode } : {}),correlationId,})returndiff --git a/control-api/src/services/tracing/administrativeEvents.ts b/control-api/src/services/tracing/administrativeEvents.tsindex 5db1f66e5..bfc2db3d9 100644--- a/control-api/src/services/tracing/administrativeEvents.ts+++ b/control-api/src/services/tracing/administrativeEvents.ts@@ -59,6 +59,13 @@ export class AdministrativePrincipalBindingInvariantError extends Error {}}+function isHccLinkedOutcome(+ principal: AdministrativeEventSubmitterPrincipalV1 | ControlApiLocalAdministrativePrincipalV1,+ input: AdministrativeEventInputV1+): boolean {+ return principal.kind === 'hcc_internal_control' && input.kind === 'linked_outcome'+}+function assertPrincipalBinding(principal: AdministrativeEventSubmitterPrincipalV1 | ControlApiLocalAdministrativePrincipalV1,binding: AdministrativeServerBindingV1,@@ -195,7 +202,14 @@ export class AdministrativeEventService {serviceSub: principal.serviceSub,sourceKind: principal.kind,sourceEventId: input.sourceEventId,- occurredAt: occurredAtIso,+ // HCC linked outcomes are state facts keyed by operation, generation and+ // outcome. HCC holds no stable observation time (it re-observes the+ // outcome on every reconcile pass and after every restart), so hashing+ // occurredAt turned each re-observation into an idempotency conflict+ // (#327). Storage keeps the first occurredAt; reasonCode, sourceStatusRef,+ // payload and the server-resolved binding still distinguish a genuinely+ // different claim. Every other submitter keeps occurredAt in the hash.+ ...(isHccLinkedOutcome(principal, input) ? {} : { occurredAt: occurredAtIso }),kind: input.kind,reasonCode: input.reasonCode ?? null,sourceStatusRef: input.sourceStatusRef ?? null,diff --git a/control-api/src/services/tracing/append.ts b/control-api/src/services/tracing/append.tsindex 668cf5932..c8ed51f75 100644--- a/control-api/src/services/tracing/append.ts+++ b/control-api/src/services/tracing/append.ts@@ -100,13 +100,35 @@ export class TracingIdempotencyConflictError extends Error {}}+/**+ * The code stays `unsafe_tracing_input` for every reason; only the message+ * differs. The server-owned/monetary message used to cover all three, which+ * misdescribed a field such as `gfs_subject` as authority smuggling (#328).+ * - `not_permitted`: a field or payload key outside the allowlist.+ * - `invalid_value`: an allowlisted key whose value has the wrong type,+ * format or size.+ */+export type UnsafeTracingInputReason =+ | 'server_owned_or_monetary'+ | 'not_permitted'+ | 'invalid_value'++const UNSAFE_TRACING_INPUT_MESSAGES: Record<UnsafeTracingInputReason, string> = {+ server_owned_or_monetary: 'tracing input contains a server-owned or monetary field',+ not_permitted: 'tracing input field is not permitted',+ invalid_value: 'tracing input value is not accepted',+}+export class UnsafeTracingInputError extends Error {readonly code = 'unsafe_tracing_input'readonly status = 400readonly statusCode = 400- constructor(readonly field: string) {- super(`tracing input contains a server-owned or monetary field: ${field}`)+ constructor(+ readonly field: string,+ readonly reason: UnsafeTracingInputReason = 'server_owned_or_monetary'+ ) {+ super(`${UNSAFE_TRACING_INPUT_MESSAGES[reason]}: ${field}`)this.name = 'UnsafeTracingInputError'}}@@ -233,39 +255,41 @@ export function assertNoClientAuthority(value: unknown, path = 'input'): void {export function assertSafeEventPayload(value: unknown): void {if (value === undefined) returnif (!value || typeof value !== 'object' || Array.isArray(value)) {- throw new UnsafeTracingInputError('input.payload')+ throw new UnsafeTracingInputError('input.payload', 'invalid_value')}for (const [key, fieldValue] of Object.entries(value as Record<string, unknown>)) {+ if (!SAFE_METADATA_KEYS.has(key)) {+ throw new UnsafeTracingInputError(`input.payload.${key}`, 'not_permitted')+ }if (- !SAFE_METADATA_KEYS.has(key) ||!['string', 'number'].includes(typeof fieldValue) ||(typeof fieldValue === 'number' && !Number.isFinite(fieldValue))) {- throw new UnsafeTracingInputError(`input.payload.${key}`)+ throw new UnsafeTracingInputError(`input.payload.${key}`, 'invalid_value')}if (key === 'tool_kind' &&!['internal_tool', 'mcp_server_tool', 'workflow'].includes(String(fieldValue))) {- throw new UnsafeTracingInputError('input.payload.tool_kind')+ throw new UnsafeTracingInputError('input.payload.tool_kind', 'invalid_value')}if (key === 'tool_source_ref' &&(typeof fieldValue !== 'string' || fieldValue.length === 0 || fieldValue.length > 128)) {- throw new UnsafeTracingInputError('input.payload.tool_source_ref')+ throw new UnsafeTracingInputError('input.payload.tool_source_ref', 'invalid_value')}if (key === 'target_label' &&(typeof fieldValue !== 'string' || !/^[A-Za-z0-9._-]{3,64}$/.test(fieldValue))) {- throw new UnsafeTracingInputError('input.payload.target_label')+ throw new UnsafeTracingInputError('input.payload.target_label', 'invalid_value')}if (key === 'target_principal_kind' &&!['operator', 'host', 'context', 'service'].includes(String(fieldValue))) {- throw new UnsafeTracingInputError('input.payload.target_principal_kind')+ throw new UnsafeTracingInputError('input.payload.target_principal_kind', 'invalid_value')}if (key === 'target_principal_ref' &&@@ -274,14 +298,14 @@ export function assertSafeEventPayload(value: unknown): void {fieldValue.length > 256 ||!/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(fieldValue))) {- throw new UnsafeTracingInputError('input.payload.target_principal_ref')+ throw new UnsafeTracingInputError('input.payload.target_principal_ref', 'invalid_value')}}const payload = value as Record<string, unknown>const targetPrincipalKind = payload.target_principal_kindconst targetPrincipalRef = payload.target_principal_refif ((targetPrincipalKind === undefined) !== (targetPrincipalRef === undefined)) {- throw new UnsafeTracingInputError('input.payload.target_principal')+ throw new UnsafeTracingInputError('input.payload.target_principal', 'invalid_value')}if (typeof targetPrincipalKind === 'string' && typeof targetPrincipalRef === 'string') {const hasCanonicalPrefix =@@ -289,11 +313,11 @@ export function assertSafeEventPayload(value: unknown): void {(targetPrincipalKind !== 'operator' &&targetPrincipalRef.startsWith(`${targetPrincipalKind}:`))if (!hasCanonicalPrefix) {- throw new UnsafeTracingInputError('input.payload.target_principal_ref')+ throw new UnsafeTracingInputError('input.payload.target_principal_ref', 'invalid_value')}}if (Buffer.byteLength(JSON.stringify(value), 'utf8') > 16_384) {- throw new UnsafeTracingInputError('input.payload')+ throw new UnsafeTracingInputError('input.payload', 'invalid_value')}}diff --git a/control-api/src/services/tracing/contracts.ts b/control-api/src/services/tracing/contracts.tsindex 49a8b606f..bd2edd30e 100644--- a/control-api/src/services/tracing/contracts.ts+++ b/control-api/src/services/tracing/contracts.ts@@ -168,6 +168,8 @@ export interface InfrastructureTelemetryEventInputV1 extends SourceOccurrenceInpname: stringnamespace: stringgeneration?: number+ /** metadata.uid of the observed Host; a recreated Host with the same name must not bind. */+ uid?: string}/** Non-authoritative lookup hint; control-api derives the WorkflowRecipe binding. */workflowRunLookupReference?: { runId: string }diff --git a/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts b/control-api/src/services/tracing/hccHealthTransitionBindingResolver.tsindex 9ee62c257..a0da0b6d0 100644--- a/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts+++ b/control-api/src/services/tracing/hccHealthTransitionBindingResolver.ts@@ -90,7 +90,11 @@ export class HccHealthTransitionBindingResolver implements InfrastructureWorkloahost.kind !== 'Host' ||metadata?.name !== reference.name ||metadata.namespace !== reference.namespace ||- (reference.generation !== undefined && metadata.generation !== reference.generation)+ (reference.generation !== undefined && metadata.generation !== reference.generation) ||+ // A Host deleted and recreated under the same name restarts at generation 1;+ // only the uid tells the objects apart, so an event observed on the old+ // object must not bind to its successor (#691).+ (reference.uid !== undefined && metadata.uid !== reference.uid)) {return null}diff --git a/control-api/src/services/tracing/routeSubmissionService.ts b/control-api/src/services/tracing/routeSubmissionService.tsindex 5a34e6917..19b7cc5f1 100644--- a/control-api/src/services/tracing/routeSubmissionService.ts+++ b/control-api/src/services/tracing/routeSubmissionService.ts@@ -190,7 +190,9 @@ function validateShape(`events[${index}]`)for (const field of Object.keys(event)) {- if (!allowed.has(field)) throw new UnsafeTracingInputError(`events[${index}].${field}`)+ if (!allowed.has(field)) {+ throw new UnsafeTracingInputError(`events[${index}].${field}`, 'not_permitted')+ }}}@@ -301,7 +303,10 @@ function normalizeInfra(typeof (hostLookupReference as Record<string, unknown>).name !== 'string' ||typeof (hostLookupReference as Record<string, unknown>).namespace !== 'string' ||((hostLookupReference as Record<string, unknown>).generation !== undefined &&- typeof (hostLookupReference as Record<string, unknown>).generation !== 'number'))+ typeof (hostLookupReference as Record<string, unknown>).generation !== 'number') ||+ ((hostLookupReference as Record<string, unknown>).uid !== undefined &&+ (typeof (hostLookupReference as Record<string, unknown>).uid !== 'string' ||+ (hostLookupReference as Record<string, unknown>).uid === '')))) {throw new InvalidTracingInputError(`events[${index}].hostLookupReference must be a Host lookup reference`diff --git a/control-api/test/http.errorHandler.clientErrorCode.test.ts b/control-api/test/http.errorHandler.clientErrorCode.test.tsnew file mode 100644index 000000000..09731b17d--- /dev/null+++ b/control-api/test/http.errorHandler.clientErrorCode.test.ts@@ -0,0 +1,56 @@+import { describe, expect, it } from 'vitest'+import express from 'express'+import request from 'supertest'+import { clerumErrorHandler } from '../src/http/errorHandler.js'+import {+ TracingIdempotencyConflictError,+ UnsafeTracingInputError,+} from '../src/services/tracing/append.js'+import { GovernedReadInvalidQueryError } from '../src/services/tracing/governedEventReadService.js'+import {+ InvalidTracingInputError,+ TracingBindingUnavailableError,+} from '../src/services/tracing/routeSubmissionService.js'++/**+ * Internal reporters (HCC) must tell a deterministic tracing rejection, which+ * they must not retry, from a transient one (#326/#327). The global handler+ * exposes a stable `code` for an allowlist of our own error types only.+ */+function appThrowing(err: unknown) {+ const app = express()+ app.post('/boom', (_req, _res, next) => next(err))+ app.use(clerumErrorHandler)+ return app+}++describe('clerumErrorHandler — machine-readable code on allowlisted 4xx', () => {+ it.each([+ [+ new TracingIdempotencyConflictError('administrative', 'hcc_internal_control', 'e-1'),+ 409,+ 'tracing_idempotency_conflict',+ ],+ [new UnsafeTracingInputError('payload.gfs_subject'), 400, 'unsafe_tracing_input'],+ [new InvalidTracingInputError('events[0].kind is not supported'), 400, 'invalid_tracing_input'],+ ])('forwards %s with its code', async (err, status, code) => {+ const res = await request(appThrowing(err)).post('/boom')++ expect(res.status).toBe(status)+ expect(res.body).toMatchObject({ code, error: expect.any(String) })+ expect(res.body.correlationId).toEqual(expect.any(String))+ })++ it.each([+ [new TracingBindingUnavailableError('operation', 0), 403],+ [new GovernedReadInvalidQueryError('cursor is not valid'), 400],+ [{ code: 409 }, 409],+ ])('keeps the body without code for a non-allowlisted 4xx (%s)', async (err, status) => {+ const res = await request(appThrowing(err)).post('/boom')++ // Liveness: the 4xx branch ran (status forwarded, correlation id present).+ expect(res.status).toBe(status)+ expect(res.body.correlationId).toEqual(expect.any(String))+ expect(res.body).not.toHaveProperty('code')+ })+})diff --git a/control-api/test/routes.tracingSubmissionBoundary.test.ts b/control-api/test/routes.tracingSubmissionBoundary.test.tsindex ad56b1b28..24ff23150 100644--- a/control-api/test/routes.tracingSubmissionBoundary.test.ts+++ b/control-api/test/routes.tracingSubmissionBoundary.test.ts@@ -3,9 +3,18 @@ import express from 'express'import jwt from 'jsonwebtoken'import request from 'supertest'import { config } from '../src/config.js'+import { clerumErrorHandler } from '../src/http/errorHandler.js'import { createInternalAdministrativeEventsRouter } from '../src/routes/internal/administrativeEvents.js'import { createInternalAgentRunEventsRouter } from '../src/routes/internal/agentRunEvents.js'import { createInternalInfrastructureTelemetryEventsRouter } from '../src/routes/internal/infrastructureTelemetryEvents.js'+import {+ TracingIdempotencyConflictError,+ UnsafeTracingInputError,+} from '../src/services/tracing/append.js'+import {+ InvalidTracingInputError,+ TracingBindingUnavailableError,+} from '../src/services/tracing/routeSubmissionService.js'import { issueMcpHostAccessJwt } from '../src/utils/auth/mcpHostJwtToken.js'function signInternalControl(issuer: 'hcc' | 'wrc'): string {@@ -283,3 +292,72 @@ describe('internal tracing submission routers', () => {expect(service.submit).not.toHaveBeenCalled()})})++// HCC settles a submission without retry only on these (status, code) pairs+// (#326), so the code has to survive the real router and the global handler.+describe('internal tracing submission routers — rejection code on the wire', () => {+ type RejectingService = { submit: () => Promise<never> }+ function rejectingApp(router: (service: RejectingService) => express.Router, err: unknown) {+ const service = { submit: vi.fn<() => Promise<never>>(async () => Promise.reject(err)) }+ const app = express()+ app.use(router(service))+ app.use(clerumErrorHandler)+ return { app, service }+ }++ it.each([+ [+ '/internal/tracing/administrative-events',+ createInternalAdministrativeEventsRouter,+ { kind: 'linked_outcome', operationId: 'operation-1' },+ new TracingIdempotencyConflictError('administrative', 'hcc_internal_control', 'e-1'),+ 409,+ 'tracing_idempotency_conflict',+ ],+ [+ '/internal/tracing/infrastructure-telemetry-events',+ createInternalInfrastructureTelemetryEventsRouter,+ { telemetryType: 'reconcile_outcome', sourceEventId: 'reconcile-1' },+ new UnsafeTracingInputError('input.payload.gfs_subject', 'not_permitted'),+ 400,+ 'unsafe_tracing_input',+ ],+ [+ '/internal/tracing/infrastructure-telemetry-events',+ createInternalInfrastructureTelemetryEventsRouter,+ { telemetryType: 'reconcile_outcome', sourceEventId: 'reconcile-1' },+ new InvalidTracingInputError('events[0].hostLookupReference.uid must be a string'),+ 400,+ 'invalid_tracing_input',+ ],+ ] as const)('%s answers %#: status and code', async (path, router, event, err, status, code) => {+ const { app, service } = rejectingApp(router, err)++ const response = await request(app)+ .post(path)+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)+ .send({ events: [event] })++ expect(service.submit).toHaveBeenCalledOnce()+ expect(response.status).toBe(status)+ expect(response.body).toMatchObject({ code, correlationId: expect.any(String) })+ })++ it('keeps a binding 403 without a code, so HCC keeps retrying it', async () => {+ const { app, service } = rejectingApp(+ createInternalAdministrativeEventsRouter,+ new TracingBindingUnavailableError('operation', 0)+ )++ const response = await request(app)+ .post('/internal/tracing/administrative-events')+ .set('Authorization', `Bearer ${signInternalControl('hcc')}`)+ .send({ events: [{ kind: 'linked_outcome', operationId: 'operation-1' }] })++ // Liveness: the service rejected and the handler's 4xx branch answered.+ expect(service.submit).toHaveBeenCalledOnce()+ expect(response.status).toBe(403)+ expect(response.body.correlationId).toEqual(expect.any(String))+ expect(response.body).not.toHaveProperty('code')+ })+})diff --git a/control-api/test/services.firstVerticalBindingResolvers.test.ts b/control-api/test/services.firstVerticalBindingResolvers.test.tsindex b1a475f11..78047388c 100644--- a/control-api/test/services.firstVerticalBindingResolvers.test.ts+++ b/control-api/test/services.firstVerticalBindingResolvers.test.ts@@ -310,6 +310,50 @@ describe('first vertical trusted binding resolvers', () => {).rejects.toMatchObject({ code: HCC_HEALTH_TRANSITION_BINDING_BLOCKER, status: 403 })})+ it('binds a reconcile outcome only to the Host object whose uid it observed (#691)', async () => {+ // Same name, namespace and generation 1: a recreated Host differs only by uid.+ const hostLookup = {+ getResource: vi.fn().mockResolvedValue({+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'Host',+ metadata: { name: 'chatllm', namespace: 'mcp-host', uid: 'new-uid', generation: 1 },+ }),+ }+ const principal = {+ kind: 'hcc_internal_control',+ sourceService: 'host-context-controller',+ serviceSub: 'hcc-provisioner',+ credentialId: 'hcc-1',+ resourceAuthority: 'hcc_managed',+ allowedTelemetryTypes: ['reconcile_outcome'],+ } as const+ const resolver = new HccHealthTransitionBindingResolver(hostLookup)+ const event = (uid?: string) => ({+ sourceEventId: `reconcile-${uid ?? 'no-uid'}`,+ occurredAt: NOW,+ telemetryType: 'reconcile_outcome' as const,+ hostLookupReference: {+ name: 'chatllm',+ namespace: 'mcp-host',+ generation: 1,+ ...(uid === undefined ? {} : { uid }),+ },+ payload: { status: 'succeeded', reason_code: 'ready' },+ })++ await expect(resolver.resolve(principal, event('new-uid'))).resolves.toMatchObject({+ kubernetesUid: 'new-uid',+ metadataGeneration: 1,+ })+ await expect(resolver.resolve(principal, event('old-uid'))).resolves.toBeNull()+ // A reference without uid (HCC before this change) still binds.+ await expect(resolver.resolve(principal, event())).resolves.toMatchObject({+ kubernetesUid: 'new-uid',+ })+ // Liveness: the authoritative Host read ran for all three references.+ expect(hostLookup.getResource).toHaveBeenCalledTimes(3)+ })+it('resolves WRC infrastructure batches from authoritative workflow rows in one query', async () => {const query = vi.fn().mockResolvedValue({rows: [diff --git a/control-api/test/services.routeTracingSubmission.test.ts b/control-api/test/services.routeTracingSubmission.test.tsindex 35869ea49..f7524533f 100644--- a/control-api/test/services.routeTracingSubmission.test.ts+++ b/control-api/test/services.routeTracingSubmission.test.ts@@ -24,10 +24,18 @@ import type {TracingTransactionRunner,} from '../src/services/tracing/contracts.js'import {+ InvalidTracingInputError,RouteTracingSubmissionService,TracingBindingUnavailableError,} from '../src/services/tracing/routeSubmissionService.js'-import { NOW, adminBinding, agentBinding, agentInput } from './services.tracingFixtures.js'+import {+ NOW,+ adminBinding,+ agentBinding,+ agentInput,+ infraBinding,+ infraPrincipal,+} from './services.tracingFixtures.js'const workflowPrincipal: AgentRunEventSubmitterPrincipalV1 = {kind: 'wrc_internal_control',@@ -366,4 +374,84 @@ describe('route tracing submission facade', () => {expect.any(Number))})++ it.each([+ ['accepts', 'host-uid-1', 1],+ ['rejects', 42, 0],+ ['rejects', '', 0],+ ] as const)(+ '%s a Host lookup reference uid of %o (#691)',+ async (_verdict, uid, expectedResolves) => {+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')+ const h = transactionHarness()+ const resolve = vi.fn().mockResolvedValue(infraBinding)+ const appendManyInTransaction = vi.fn().mockResolvedValue([appendResult('accepted')])+ const service = new RouteTracingSubmissionService({+ transaction: h.transaction,+ infrastructureWorkloadBindingResolver: { resolve },+ infrastructureTelemetryAppender: { appendManyInTransaction },+ })+ const submission = service.submit({+ principal: infraPrincipal,+ events: [+ {+ sourceEventId: 'reconcile-1',+ occurredAt: NOW,+ telemetryType: 'reconcile_outcome',+ hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1, uid },+ payload: { status: 'succeeded', reason_code: 'ready' },+ },+ ],+ })++ if (expectedResolves === 1) {+ await expect(submission).resolves.toEqual({ accepted: 1, replayed: 0 })+ expect(resolve.mock.calls[0]![1].hostLookupReference).toEqual({+ name: 'chatllm',+ namespace: 'mcp-host',+ generation: 1,+ uid,+ })+ } else {+ await expect(submission).rejects.toBeInstanceOf(InvalidTracingInputError)+ // Liveness: the rejection was counted, so validation ran on this event.+ expect(rejected).toHaveBeenCalled()+ }+ expect(resolve).toHaveBeenCalledTimes(expectedResolves)+ }+ )++ it('names an unknown event field as not permitted, not as server-owned (#328)', async () => {+ const rejected = vi.spyOn(governedTraceRejectedTotal, 'inc')+ const h = transactionHarness()+ const resolve = vi.fn().mockResolvedValue(infraBinding)+ const service = new RouteTracingSubmissionService({+ transaction: h.transaction,+ infrastructureWorkloadBindingResolver: { resolve },+ infrastructureTelemetryAppender: { appendManyInTransaction: vi.fn() },+ })++ await expect(+ service.submit({+ principal: infraPrincipal,+ events: [+ {+ sourceEventId: 'reconcile-1',+ occurredAt: NOW,+ telemetryType: 'reconcile_outcome',+ hostLookupReference: { name: 'chatllm', namespace: 'mcp-host', generation: 1 },+ payload: { status: 'succeeded', reason_code: 'ready' },+ gfsEvidence: 'rotated',+ },+ ],+ })+ ).rejects.toMatchObject({+ code: 'unsafe_tracing_input',+ reason: 'not_permitted',+ message: 'tracing input field is not permitted: events[0].gfsEvidence',+ })+ // Liveness: the rejection was counted, so validation ran on this event.+ expect(rejected).toHaveBeenCalled()+ expect(resolve).not.toHaveBeenCalled()+ })})diff --git a/control-api/test/services.tracingAppends.test.ts b/control-api/test/services.tracingAppends.test.tsindex 46d5731cc..291bbf150 100644--- a/control-api/test/services.tracingAppends.test.ts+++ b/control-api/test/services.tracingAppends.test.ts@@ -16,6 +16,7 @@ import {import {TracingIdempotencyConflictError,UnsafeTracingInputError,+ assertNoClientAuthority,assertSafeEventPayload,} from '../src/services/tracing/append.js'import type {@@ -369,6 +370,75 @@ describe('governed append idempotency', () => {).resolves.toMatchObject({ kind: 'replayed' })})+ describe('HCC linked outcomes re-observed at a later time (#327)', () => {+ const hccAdminPrincipal = {+ kind: 'hcc_internal_control',+ sourceService: 'host-context-controller',+ serviceSub: 'hcc-provisioner',+ credentialId: 'hcc-admin',+ allowedKinds: ['linked_outcome'],+ } as const satisfies AdministrativeEventSubmitterPrincipalV1+ const hccOutcome = {+ kind: 'linked_outcome' as const,+ sourceEventId: `hcc-admin-outcome:${adminBinding.operationId}:3:succeeded`,+ occurredAt: '2026-07-10T09:59:59.000Z',+ reasonCode: 'reconciled',+ sourceStatusRef: 'host:mcp-host/chatllm:generation=3',+ payload: { resource_class: 'Host', status: 'succeeded' },+ }+ const service = () =>+ new AdministrativeEventService({+ transaction: h.transaction,+ now: () => new Date(NOW),+ newEventId: () => EVENT_ID,+ })+ const insertCalls = () =>+ h.query.mock.calls.filter(call => String(call[0]).includes('inserted_family'))++ it('replays the same outcome with a different occurredAt and keeps the first one', async () => {+ await expect(+ service().append(hccAdminPrincipal, adminBinding, hccOutcome)+ ).resolves.toMatchObject({ kind: 'accepted', accepted: 1 })+ await expect(+ service().append(hccAdminPrincipal, adminBinding, {+ ...hccOutcome,+ occurredAt: '2026-07-10T10:05:00.000Z',+ })+ ).resolves.toMatchObject({ kind: 'replayed', replayed: 1, eventId: EVENT_ID })++ const inserts = insertCalls()+ expect(inserts).toHaveLength(1)+ expect(JSON.stringify(inserts[0][1])).toContain(hccOutcome.occurredAt)+ })++ it('still conflicts when the same outcome key claims a different reasonCode', async () => {+ await service().append(hccAdminPrincipal, adminBinding, hccOutcome)++ await expect(+ service().append(hccAdminPrincipal, adminBinding, {+ ...hccOutcome,+ occurredAt: '2026-07-10T10:05:00.000Z',+ reasonCode: 'reconcile_exception',+ })+ ).rejects.toBeInstanceOf(TracingIdempotencyConflictError)+ })++ it('keeps occurredAt in the digest for every other linked_outcome submitter', async () => {+ const wrcOutcome = { ...hccOutcome, sourceEventId: 'wrc-outcome-1' }+ await expect(+ service().append(adminPrincipal, adminBinding, wrcOutcome)+ ).resolves.toMatchObject({ kind: 'accepted' })++ await expect(+ service().append(adminPrincipal, adminBinding, {+ ...wrcOutcome,+ occurredAt: '2026-07-10T10:05:00.000Z',+ })+ ).rejects.toBeInstanceOf(TracingIdempotencyConflictError)+ expect(insertCalls()).toHaveLength(1)+ })+ })+it('does not collide events from different source services with the same kind and source event id', async () => {const secondServicePrincipal = { ...agentPrincipal, sourceService: 'mcp-host-secondary' }const lockIdentities: string[] = []@@ -457,6 +527,51 @@ describe('governed append idempotency', () => {})describe('server-owned tracing fields', () => {+ it('accepts HCC GFS evidence as a transition and names a non-allowlisted key as such (#328)', () => {+ expect(() =>+ assertSafeEventPayload({+ resource_class: 'Host',+ reason_code: 'ready',+ status: 'succeeded',+ phase: 'deployed',+ state: 'ready',+ transition: 'gfs_token:reused',+ })+ ).not.toThrow()+ expect(() =>+ assertSafeEventPayload({+ transition: 'gfs_token:reused',+ gfs_subject: 'host:1st:mcp-host/chatllm',+ })+ ).toThrow(+ expect.objectContaining({+ code: 'unsafe_tracing_input',+ reason: 'not_permitted',+ message: 'tracing input field is not permitted: input.payload.gfs_subject',+ })+ )+ expect(() => assertSafeEventPayload({ transition: { outcome: 'reused' } })).toThrow(+ expect.objectContaining({+ code: 'unsafe_tracing_input',+ reason: 'invalid_value',+ message: 'tracing input value is not accepted: input.payload.transition',+ })+ )+ expect(() => assertSafeEventPayload({ target_label: 'Label with spaces' })).toThrow(+ expect.objectContaining({+ reason: 'invalid_value',+ message: 'tracing input value is not accepted: input.payload.target_label',+ })+ )+ expect(() => assertNoClientAuthority({ payload: { sourceService: 'hcc' } })).toThrow(+ expect.objectContaining({+ code: 'unsafe_tracing_input',+ message:+ 'tracing input contains a server-owned or monetary field: input.payload.sourceService',+ })+ )+ })+it('accepts a bounded administrator label but rejects arbitrary display text', () => {expect(() => assertSafeEventPayload({ target_label: 'deleted_admin' })).not.toThrow()expect(() => assertSafeEventPayload({ target_label: 'Deleted admin with spaces' })).toThrow(diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex ac10a721e..b2905694f 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.340","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.340","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 921d081c5..81dfb1b9c 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.340","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/administrativeOutcomeReporter.test.ts b/host-context-controller/src/administrativeOutcomeReporter.test.tsindex e9e09e8f6..7aaf39516 100644--- a/host-context-controller/src/administrativeOutcomeReporter.test.ts+++ b/host-context-controller/src/administrativeOutcomeReporter.test.ts@@ -1,8 +1,10 @@-import { describe, expect, it, vi } from 'vitest'+import { beforeEach, describe, expect, it, vi } from 'vitest'import {+ type AdministrativeHostOutcomeProjection,BoundedAdministrativeOutcomeReporter,createAdministrativeOutcomeReporter,} from './administrativeOutcomeReporter'+import { administrativeOutcomeReporterTotal } from './metrics'describe('createAdministrativeOutcomeReporter', () => {it('does not construct a reporter when governed tracing is disabled', () => {@@ -121,3 +123,208 @@ describe('BoundedAdministrativeOutcomeReporter', () => {expect(fetchFn).toHaveBeenCalledOnce()})})++function outcome(sourceEventId: string): AdministrativeHostOutcomeProjection {+ return {+ sourceEventId,+ occurredAt: '2026-09-18T10:00:00.000Z',+ hostRef: { name: 'chatllm', namespace: 'mcp-host', generation: 7 },+ outcome: 'succeeded',+ reasonCode: 'reconciled',+ }+}++function failedResponse(status: number, body?: unknown): Response {+ return {+ ok: false,+ status,+ json: async () => {+ if (body === undefined) throw new SyntaxError('Unexpected end of JSON input')+ return body+ },+ } as unknown as Response+}++async function counted(result: string): Promise<number> {+ const metric = await administrativeOutcomeReporterTotal.get()+ return metric.values.find(value => value.labels.result === result)?.value ?? 0+}++function submittedIds(fetchFn: typeof fetch): string[] {+ return vi.mocked(fetchFn).mock.calls.map(call => {+ const body = JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> }+ return body.events[0]!.sourceEventId+ })+}++// Default retry delays with random() = 0 are 25 ms then 50 ms.+const settle = () => new Promise(resolve => setTimeout(resolve, 150))++describe('BoundedAdministrativeOutcomeReporter — once per process (#327, #326)', () => {+ beforeEach(() => administrativeOutcomeReporterTotal.reset())++ function reporterWith(+ fetchFn: typeof fetch,+ bounds: { capacity?: number; dedupeCapacity?: number } = {}+ ) {+ return new BoundedAdministrativeOutcomeReporter({+ baseUrl: 'http://control-api.test:8090',+ signToken: () => 'signed',+ fetchFn,+ random: () => 0,+ ...bounds,+ })+ }++ it('deduplicates a key while its first copy is queued or waiting for a retry', async () => {+ const fetchFn = vi.fn().mockResolvedValue(failedResponse(500)) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('in-flight'))+ reporter.enqueueHostOutcome(outcome('in-flight'))+ // The first attempt has failed; the retry fires at 25 ms.+ await new Promise(resolve => setTimeout(resolve, 10))+ expect(fetchFn).toHaveBeenCalledOnce()+ reporter.enqueueHostOutcome(outcome('in-flight'))+ await settle()++ // Liveness: the one queued copy used its whole retry budget.+ expect(fetchFn).toHaveBeenCalledTimes(3)+ expect(await counted('retry_exhausted')).toBe(1)+ expect(await counted('deduplicated')).toBe(2)+ })++ it('sends an outcome again after it was dropped because the buffer was full', async () => {+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })++ reporter.enqueueHostOutcome(outcome('buffered'))+ reporter.enqueueHostOutcome(outcome('dropped'))+ await settle()+ expect(await counted('buffer_full')).toBe(1)+ reporter.enqueueHostOutcome(outcome('dropped'))+ await settle()++ expect(submittedIds(fetchFn)).toEqual(['buffered', 'dropped'])+ expect(await counted('deduplicated')).toBe(0)+ })++ it('keeps a key that is seen on every pass ahead of keys seen once', async () => {+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })++ for (const id of ['repeated', 'once', 'repeated', 'newer', 'repeated']) {+ reporter.enqueueHostOutcome(outcome(id))+ await settle()+ }++ // 'newer' evicted 'once', not 'repeated', because 'repeated' was seen again.+ expect(submittedIds(fetchFn)).toEqual(['repeated', 'once', 'newer'])+ expect(await counted('deduplicated')).toBe(2)+ })++ it('requires the dedupe bound to exceed the queue capacity', () => {+ const fetchFn = vi.fn() as unknown as typeof fetch++ expect(() => reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 4 })).toThrow(+ 'dedupeCapacity must be an integer greater than capacity'+ )+ expect(reporterWith(fetchFn, { capacity: 4, dedupeCapacity: 5 })).toBeInstanceOf(+ BoundedAdministrativeOutcomeReporter+ )+ })++ it('sends an accepted outcome once however often the reconciler re-observes it', async () => {+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))+ await settle()+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:succeeded'))+ await settle()++ expect(fetchFn).toHaveBeenCalledOnce()+ expect(await counted('accepted')).toBe(1)+ expect(await counted('deduplicated')).toBe(2)+ })++ it('settles a 409 idempotency conflict without retry and deduplicates the key afterwards', async () => {+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {+ const id = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })+ .events[0]!.sourceEventId+ return id === 'conflicting'+ ? failedResponse(409, { code: 'tracing_idempotency_conflict', error: 'conflict' })+ : failedResponse(500, { error: 'Internal server error' })+ }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('conflicting'))+ reporter.enqueueHostOutcome(outcome('transient'))+ await settle()+ reporter.enqueueHostOutcome(outcome('conflicting'))+ await settle()++ const ids = submittedIds(fetchFn)+ // Liveness: the transient failure on another key is retried twice.+ expect(ids.filter(id => id === 'transient')).toHaveLength(3)+ expect(ids.filter(id => id === 'conflicting')).toHaveLength(1)+ expect(await counted('conflict')).toBe(1)+ expect(await counted('deduplicated')).toBe(1)+ })++ it('sends an outcome again after its retries were exhausted', async () => {+ const fetchFn = vi+ .fn()+ .mockResolvedValueOnce(failedResponse(500))+ .mockResolvedValueOnce(failedResponse(500))+ .mockResolvedValueOnce(failedResponse(500))+ .mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:failed'))+ await settle()+ expect(await counted('retry_exhausted')).toBe(1)+ reporter.enqueueHostOutcome(outcome('hcc-admin-outcome:op-1:7:failed'))+ await settle()++ expect(fetchFn).toHaveBeenCalledTimes(4)+ expect(await counted('accepted')).toBe(1)+ expect(await counted('deduplicated')).toBe(0)+ })++ it.each([+ ['a 403 binding not yet visible', failedResponse(403, { error: 'Forbidden' })],+ ['a 409 without a body', failedResponse(409)],+ ['a 409 without a code', failedResponse(409, { error: 'Conflict' })],+ ['a 400 without a code', failedResponse(400, { error: 'Bad Request' })],+ ])('keeps retrying %s', async (_label, response) => {+ const fetchFn = vi.fn().mockResolvedValue(response) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn)++ reporter.enqueueHostOutcome(outcome('retryable'))+ await settle()++ expect(fetchFn).toHaveBeenCalledTimes(3)+ expect(await counted('retry_exhausted')).toBe(1)+ expect(await counted('conflict')).toBe(0)+ expect(await counted('rejected')).toBe(0)+ })++ it('forgets the oldest settled outcome once the dedupe bound is reached', async () => {+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const reporter = reporterWith(fetchFn, { capacity: 1, dedupeCapacity: 2 })++ for (const id of ['first', 'second', 'third']) {+ reporter.enqueueHostOutcome(outcome(id))+ await settle()+ }+ reporter.enqueueHostOutcome(outcome('third'))+ reporter.enqueueHostOutcome(outcome('first'))+ await settle()++ // 'third' is still remembered (witness); 'first' was evicted and is sent again.+ expect(submittedIds(fetchFn)).toEqual(['first', 'second', 'third', 'first'])+ expect(await counted('deduplicated')).toBe(1)+ })+})diff --git a/host-context-controller/src/administrativeOutcomeReporter.ts b/host-context-controller/src/administrativeOutcomeReporter.tsindex 8e1503c2b..f14e252d7 100644--- a/host-context-controller/src/administrativeOutcomeReporter.ts+++ b/host-context-controller/src/administrativeOutcomeReporter.ts@@ -1,6 +1,7 @@import { BoundedOffPathReporter } from './boundedOffPathReporter'import { config } from './config'import { administrativeOutcomeReporterTotal } from './metrics'+import { throwForFailedSubmit } from './reporterHttpFailure'import { signInternalControlJwt } from './utils/internalControlSigner'export type AdministrativeHostOutcomeProjection = {@@ -24,12 +25,14 @@ export type AdministrativeOutcomeReporterDependencies = {retryLimit?: numbertimeoutMs?: numberrandom?: () => number+ dedupeCapacity?: number}const DEFAULT_CAPACITY = 64const DEFAULT_RETRY_LIMIT = 2const DEFAULT_TIMEOUT_MS = 1_000const DEFAULT_STOP_TIMEOUT_MS = 1_500+const DEFAULT_DEDUPE_CAPACITY = 1_024export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcomeReporter {private readonly queue: BoundedOffPathReporter<AdministrativeHostOutcomeProjection>@@ -37,12 +40,31 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcoprivate readonly baseUrl: stringprivate readonly signToken: () => stringprivate readonly fetchFn: typeof fetch+ private readonly dedupeCapacity: number+ // The reconciler re-observes the same outcome on every pass (3-4 fleet+ // passes per 5 minutes) and its sourceEventId is deterministic, so each+ // fact is sent once per process (#327). A key stays here while it is queued,+ // in flight, accepted or rejected as terminal; only a drop removes it.+ // Iteration order is least recently seen first, which is the eviction+ // order. Past `dedupeCapacity` distinct keys per pass, the oldest are+ // evicted and resent; control-api answers those resends as replays.+ private readonly seen = new Set<string>()constructor(deps: AdministrativeOutcomeReporterDependencies) {const capacity = deps.capacity ?? DEFAULT_CAPACITYif (!Number.isSafeInteger(capacity) || capacity < 1) {throw new Error('administrative outcome reporter capacity must be a positive integer')}+ const dedupeCapacity = deps.dedupeCapacity ?? DEFAULT_DEDUPE_CAPACITY+ // Up to `capacity` keys are buffered plus one in flight. A smaller set+ // would evict a key that is still queued, and the next pass would queue+ // it a second time.+ if (!Number.isSafeInteger(dedupeCapacity) || dedupeCapacity <= capacity) {+ throw new Error(+ 'administrative outcome reporter dedupeCapacity must be an integer greater than capacity'+ )+ }+ this.dedupeCapacity = dedupeCapacitythis.timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MSthis.baseUrl = (deps.baseUrl ?? config.controlApiBaseUrl).replace(/\/$/, '')this.signToken = deps.signToken ?? signInternalControlJwt@@ -55,11 +77,30 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcosubmit: projection => this.submit(projection),onEnqueued: () => administrativeOutcomeReporterTotal.inc({ result: 'enqueued' }),onAccepted: () => administrativeOutcomeReporterTotal.inc({ result: 'accepted' }),- onDrop: (_projection, reason) => administrativeOutcomeReporterTotal.inc({ result: reason }),+ onTerminal: (_projection, result) => administrativeOutcomeReporterTotal.inc({ result }),+ onDrop: (projection, reason) => {+ // A dropped outcome was never delivered; forgetting it lets the next+ // reconcile pass enqueue it again, so no outcome is lost.+ this.seen.delete(projection.sourceEventId)+ administrativeOutcomeReporterTotal.inc({ result: reason })+ },})}enqueueHostOutcome(projection: AdministrativeHostOutcomeProjection): void {+ const { sourceEventId } = projection+ if (this.seen.delete(sourceEventId)) {+ // Re-inserted as most recently seen, so a key observed on every pass is+ // not evicted behind keys observed once.+ this.seen.add(sourceEventId)+ administrativeOutcomeReporterTotal.inc({ result: 'deduplicated' })+ return+ }+ if (this.seen.size >= this.dedupeCapacity) {+ const oldest = this.seen.values().next().value+ if (oldest !== undefined) this.seen.delete(oldest)+ }+ this.seen.add(sourceEventId)this.queue.enqueue({ ...projection, hostRef: { ...projection.hostRef } })}@@ -97,9 +138,7 @@ export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcosignal: controller.signal,})- if (!response.ok) {- throw new Error(`administrative outcome submit failed with ${response.status}`)- }+ await throwForFailedSubmit(response, 'administrative outcome')} finally {clearTimeout(timeout)}diff --git a/host-context-controller/src/boundedOffPathReporter.test.ts b/host-context-controller/src/boundedOffPathReporter.test.tsindex 5b8d8a216..742568148 100644--- a/host-context-controller/src/boundedOffPathReporter.test.ts+++ b/host-context-controller/src/boundedOffPathReporter.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from 'vitest'import {BoundedOffPathReporter,type BoundedOffPathReporterOptions,+ ReporterTerminalError,} from './boundedOffPathReporter'function createReporter(overrides: Partial<BoundedOffPathReporterOptions<string>> = {}) {@@ -12,6 +13,7 @@ function createReporter(overrides: Partial<BoundedOffPathReporterOptions<string>random: () => 0,submit: vi.fn().mockResolvedValue(undefined),onAccepted: vi.fn(),+ onTerminal: vi.fn(),onDrop: vi.fn(),...overrides,})@@ -69,6 +71,25 @@ describe('BoundedOffPathReporter', () => {expect(onDrop).toHaveBeenCalledWith('event-1', 'retry_exhausted')})+ it('settles a terminal rejection once, without retry or drop (#326)', async () => {+ const submit = vi+ .fn()+ .mockRejectedValue(new ReporterTerminalError('conflict', 'submit rejected with 409'))+ const onTerminal = vi.fn()+ const onRetry = vi.fn()+ const onDrop = vi.fn()+ const reporter = createReporter({ retryLimit: 2, submit, onTerminal, onRetry, onDrop })++ reporter.enqueue('event-1')+ await new Promise(resolve => setTimeout(resolve, 150))++ // Liveness: the submission ran and its rejection was classified.+ expect(submit).toHaveBeenCalledOnce()+ expect(onTerminal).toHaveBeenCalledExactlyOnceWith('event-1', 'conflict')+ expect(onRetry).not.toHaveBeenCalled()+ expect(onDrop).not.toHaveBeenCalled()+ })+it('drains queued entries during stop and rejects later enqueue', async () => {const submit = vi.fn().mockResolvedValue(undefined)const onDrop = vi.fn()diff --git a/host-context-controller/src/boundedOffPathReporter.ts b/host-context-controller/src/boundedOffPathReporter.tsindex 9c2a221b4..ec12962d1 100644--- a/host-context-controller/src/boundedOffPathReporter.ts+++ b/host-context-controller/src/boundedOffPathReporter.ts@@ -6,6 +6,22 @@ export type BoundedReporterDropReason =| 'retry_exhausted'| 'shutdown_timeout'+export type ReporterTerminalResult = 'conflict' | 'rejected'++/**+ * A submission the receiver rejected deterministically. Resending the same+ * value yields the same answer, so the queue settles it without a retry (#326).+ */+export class ReporterTerminalError extends Error {+ constructor(+ readonly result: ReporterTerminalResult,+ message: string+ ) {+ super(message)+ this.name = 'ReporterTerminalError'+ }+}+type BufferedEntry<Value> = { value: Value; attempts: number }export type BoundedOffPathReporterOptions<Value> = {@@ -17,6 +33,7 @@ export type BoundedOffPathReporterOptions<Value> = {onEnqueued?: (value: Value) => voidonAccepted: (value: Value) => voidonRetry?: (value: Value) => void+ onTerminal: (value: Value, result: ReporterTerminalResult) => voidonDrop: (value: Value, reason: BoundedReporterDropReason) => void}@@ -115,7 +132,11 @@ export class BoundedOffPathReporter<Value> {try {await this.options.submit(entry.value)this.options.onAccepted(entry.value)- } catch {+ } catch (error) {+ if (error instanceof ReporterTerminalError) {+ this.options.onTerminal(entry.value, error.result)+ return+ }if (this.stopped || entry.attempts >= this.options.retryLimit) {this.options.onDrop(entry.value,diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 459e738ea..2f8242593 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -19,6 +19,7 @@ import { GFS_HOST_SCOPES } from './gfsHostPolicy'import { makeExpectedHostGfsSubject } from './gfsHostSubject'import type {HccInfrastructureTelemetryPayload,+ HostLookupReference,InfrastructureTelemetryReporter,} from './infrastructureTelemetryReporter'import {@@ -502,12 +503,8 @@ type RuntimeTokenProvision = {scopeHash: string}-type GfsTokenLifecycleEvidence = {- gfs_subject: string- gfs_outcome: 'minted' | 'rotated' | 'reused' | 'failed'- gfs_old_host_uid?: string- gfs_new_host_uid?: string-}+// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).+type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'type DeploymentMutationState = {lifecycle: EffectiveHostLifecycle@@ -558,7 +555,7 @@ export class HostReconciler {) => CommunicationChannelCRD[]private readonly infrastructureTelemetryReporter?: InfrastructureTelemetryReporterprivate readonly administrativeOutcomeReporter?: AdministrativeOutcomeReporter- private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleEvidence>()+ private readonly gfsTokenLifecycleEvidence = new Map<string, GfsTokenLifecycleOutcome>()// B2: whether the CC cache initial-list has completed. Defaults to false// (safe: preserves existing Deployment replicas until wired by McpServerWatcher).private ccCacheSyncedFn: () => boolean = () => false@@ -652,11 +649,7 @@ export class HostReconciler {this.infrastructureTelemetryReporter?.enqueueHealthTransition({sourceEventId: `hcc-health-transition:${this.newTelemetryOccurrenceId()}`,occurredAt,- hostLookupReference: {- name: host.name,- namespace: host.namespace,- ...(host.generation !== undefined ? { generation: host.generation } : {}),- },+ hostLookupReference: this.hostLookupReference(host),payload: { transition: `lifecycle:${lifecycle.state}`, state: lifecycle.state },})},@@ -1850,11 +1843,7 @@ export class HostReconciler {resourceName: name,refreshExpInHours,})- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {- gfs_subject: expectedGfsSubject,- gfs_outcome: 'reused',- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),- })+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'reused')const selectedRevision =rolloutPending || HostReconciler.shouldRollForRuntimeSecret(deployment, false)? existingRevision@@ -1934,11 +1923,7 @@ export class HostReconciler {await observeCreate('Secret', () =>this.coreApi.createNamespacedSecret({ namespace: host.namespace, body }))- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {- gfs_subject: expectedGfsSubject,- gfs_outcome: 'minted',- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),- })+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'minted')log.info('created mcp-host-runtime-token Secret', {host: host.name,namespace: host.namespace,@@ -1960,13 +1945,7 @@ export class HostReconciler {namespace: host.namespace,body: replaceBody,})- const oldHostUid = existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION]- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {- gfs_subject: expectedGfsSubject,- gfs_outcome: 'rotated',- ...(oldHostUid ? { gfs_old_host_uid: oldHostUid } : {}),- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),- })+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'rotated')log.info('rotated mcp-host-runtime-token Secret', {host: host.name,namespace: host.namespace,@@ -1982,13 +1961,8 @@ export class HostReconciler {}} catch (err) {lastErr = err- const subject = makeExpectedHostGfsSubject(host.namespace, host.name)- if (subject) {- this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), {- gfs_subject: subject,- gfs_outcome: 'failed',- ...(host.uid ? { gfs_new_host_uid: host.uid } : {}),- })+ if (makeExpectedHostGfsSubject(host.namespace, host.name)) {+ this.gfsTokenLifecycleEvidence.set(HostReconciler.gfsLifecycleEvidenceKey(host), 'failed')}if (err instanceof Error && err.name === 'GfsHostTokenValidationError') breakconst delayMs = 1000 * Math.pow(2, attempt - 1)@@ -2154,15 +2128,12 @@ export class HostReconciler {}}- private hostLookupReference(host: HostCRD): {- name: string- namespace: string- generation?: number- } {+ private hostLookupReference(host: HostCRD): HostLookupReference {return {name: host.name,namespace: host.namespace,...(host.generation !== undefined ? { generation: host.generation } : {}),+ ...(host.uid !== undefined ? { uid: host.uid } : {}),}}@@ -2207,16 +2178,38 @@ export class HostReconciler {private enqueueReconcileOutcome(host: HostCRD): void {const status = this.getStatus(host.name)const succeeded = status.deployed && status.ready- this.enqueueHostTelemetry(host, 'reconcile_outcome', succeeded ? 'ready' : 'not_ready', {+ this.enqueueReconcileTelemetry(host, succeeded ? 'ready' : 'not_ready', {status: succeeded ? 'succeeded' : 'failed',phase: status.deployed ? 'deployed' : 'not_deployed',state: status.ready ? 'ready' : 'not_ready',- ...(this.gfsTokenLifecycleEvidence.get(HostReconciler.gfsLifecycleEvidenceKey(host)) ?? {}),})- this.gfsTokenLifecycleEvidence.delete(HostReconciler.gfsLifecycleEvidenceKey(host))if (succeeded) this.enqueueAdministrativeOutcome(host, 'succeeded', 'reconciled')}+ /**+ * A pass that threw may not have written any status yet, so the status in+ * memory can still be the previous pass's. Report the failure itself, not+ * that status, and never an administrative `succeeded` (#327).+ */+ private enqueueReconcileExceptionOutcome(host: HostCRD): void {+ this.enqueueReconcileTelemetry(host, 'reconcile_exception', { status: 'failed' })+ }++ private enqueueReconcileTelemetry(+ host: HostCRD,+ reasonCode: string,+ payload: HccInfrastructureTelemetryPayload+ ): void {+ const evidenceKey = HostReconciler.gfsLifecycleEvidenceKey(host)+ const gfsOutcome = this.gfsTokenLifecycleEvidence.get(evidenceKey)+ this.gfsTokenLifecycleEvidence.delete(evidenceKey)+ this.enqueueHostTelemetry(host, 'reconcile_outcome', reasonCode, {+ ...payload,+ // control-api allowlists `transition`, not gfs_* keys (#328).+ ...(gfsOutcome ? { transition: `gfs_token:${gfsOutcome}` } : {}),+ })+ }+private enqueueAdministrativeOutcome(host: HostCRD,outcome: 'succeeded' | 'failed',@@ -4435,11 +4428,19 @@ export class HostReconciler {// or an administrative 'failed' outcome. Callers already treat the// rethrow as a retire (reconcileDelete → 'superseded', watch callers// only log the error), so the rethrow is preserved.+ // This exit emits no outcome, so drop the GFS evidence this pass+ // recorded. Keyed by namespace/name, a leftover would be reported by+ // the next pass, or by a recreated Host with the same name. The+ // stateless suspension path also calls reconcileCore without emitting+ // an outcome and does not clear the entry (#696).+ this.gfsTokenLifecycleEvidence.delete(+ HostReconciler.gfsLifecycleEvidenceKey(admittedHost)+ )this.observeReconcileLatency(source, 'superseded', dispatchedAt, admittedAt)throw error}this.enqueueControllerError(admittedHost, 'reconcile_exception', error)- this.enqueueReconcileOutcome(admittedHost)+ this.enqueueReconcileExceptionOutcome(admittedHost)this.observeReconcileLatency(source, 'error', dispatchedAt, admittedAt)throw error} finally {diff --git a/host-context-controller/src/infrastructureTelemetryReporter.test.ts b/host-context-controller/src/infrastructureTelemetryReporter.test.tsindex 8cf633e10..27ea9c311 100644--- a/host-context-controller/src/infrastructureTelemetryReporter.test.ts+++ b/host-context-controller/src/infrastructureTelemetryReporter.test.ts@@ -4,7 +4,11 @@ import {createInfrastructureTelemetryReporter,hccReconcileOutcomeSourceId,} from './infrastructureTelemetryReporter'-import { infrastructureTelemetryGapsTotal, infrastructureTelemetryRetriesTotal } from './metrics'+import {+ infrastructureTelemetryFlushesTotal,+ infrastructureTelemetryGapsTotal,+ infrastructureTelemetryRetriesTotal,+} from './metrics'const projection = {sourceEventId: 'hcc-health-transition:mcp-host:chatllm:7:active:1',@@ -134,7 +138,7 @@ describe('BoundedInfrastructureTelemetryReporter', () => {it('keeps reconcile identity stable for equivalent state and changes it with the outcome', () => {expect(hccReconcileOutcomeSourceId(reconcileProjection)).toMatch(- /^hcc-reconcile-outcome-v2:[0-9a-f]{64}$/+ /^hcc-reconcile-outcome-v3:[0-9a-f]{64}$/)expect(hccReconcileOutcomeSourceId(reconcileProjection)).toBe(hccReconcileOutcomeSourceId({ ...reconcileProjection })@@ -151,6 +155,108 @@ describe('BoundedInfrastructureTelemetryReporter', () => {)})+ it('gives a recreated Host object (same name and generation, new uid) new identities (#691)', () => {+ const withUid = (uid: string) => ({+ ...reconcileProjection,+ hostLookupReference: { ...reconcileProjection.hostLookupReference, uid },+ })++ // Liveness: the identity is deterministic for an identical tuple.+ expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).toBe(+ hccReconcileOutcomeSourceId(withUid('uid-new'))+ )+ expect(hccReconcileOutcomeSourceId(withUid('uid-new'))).not.toBe(+ hccReconcileOutcomeSourceId(withUid('uid-old'))+ )+ })++ it('settles a 400 unsafe_tracing_input once and keeps retrying a 500 (#326, #328)', async () => {+ infrastructureTelemetryFlushesTotal.reset()+ infrastructureTelemetryGapsTotal.reset()+ const response = (status: number, body: unknown) =>+ ({ ok: false, status, json: async () => body }) as unknown as Response+ const fetchFn = vi.fn(async (_url: unknown, init?: RequestInit) => {+ const event = (JSON.parse(String(init?.body)) as { events: Array<{ sourceEventId: string }> })+ .events[0]!+ return event.sourceEventId === 'rejected-event'+ ? response(400, { code: 'unsafe_tracing_input', error: 'unsafe' })+ : response(500, { error: 'Internal server error' })+ }) as unknown as typeof fetch+ const reporter = new BoundedInfrastructureTelemetryReporter({+ baseUrl: 'http://control-api.test:8090',+ signToken: () => 'signed-request',+ fetchFn,+ random: () => 0,+ })++ reporter.enqueue({+ ...projection,+ telemetryType: 'lifecycle_transition',+ sourceEventId: 'rejected-event',+ })+ reporter.enqueue({+ ...projection,+ telemetryType: 'lifecycle_transition',+ sourceEventId: 'transient-event',+ })+ await new Promise(resolve => setTimeout(resolve, 150))++ const ids = vi+ .mocked(fetchFn)+ .mock.calls.map(+ call =>+ (JSON.parse(String(call[1]?.body)) as { events: Array<{ sourceEventId: string }> })+ .events[0]!.sourceEventId+ )+ // Liveness: the 500 on the other event is retried twice.+ expect(ids.filter(id => id === 'transient-event')).toHaveLength(3)+ expect(ids.filter(id => id === 'rejected-event')).toHaveLength(1)+ const flushes = (await infrastructureTelemetryFlushesTotal.get()).values+ expect(flushes.find(value => value.labels.result === 'rejected')?.value).toBe(1)+ expect(flushes.find(value => value.labels.result === 'exhausted')?.value).toBe(1)+ // Both events were lost: one rejected, one after its retries.+ const gaps = (await infrastructureTelemetryGapsTotal.get()).values.map(value => value.labels)+ expect(gaps).toEqual(+ expect.arrayContaining([+ { telemetry_type: 'lifecycle_transition', reason: 'rejected' },+ { telemetry_type: 'lifecycle_transition', reason: 'retry_exhausted' },+ ])+ )+ expect(gaps).toHaveLength(2)+ })++ it('does not count a 409 idempotency conflict as an evidence gap', async () => {+ infrastructureTelemetryFlushesTotal.reset()+ infrastructureTelemetryGapsTotal.reset()+ const fetchFn = vi.fn(+ async () =>+ ({+ ok: false,+ status: 409,+ json: async () => ({ code: 'tracing_idempotency_conflict' }),+ }) as unknown as Response+ ) as unknown as typeof fetch+ const reporter = new BoundedInfrastructureTelemetryReporter({+ baseUrl: 'http://control-api.test:8090',+ signToken: () => 'signed-request',+ fetchFn,+ random: () => 0,+ })++ reporter.enqueue({+ ...projection,+ telemetryType: 'lifecycle_transition',+ sourceEventId: 'conflicting-event',+ })+ await new Promise(resolve => setTimeout(resolve, 150))++ // Liveness: the conflict was submitted once and settled.+ expect(fetchFn).toHaveBeenCalledOnce()+ const flushes = (await infrastructureTelemetryFlushesTotal.get()).values+ expect(flushes.find(value => value.labels.result === 'conflict')?.value).toBe(1)+ expect((await infrastructureTelemetryGapsTotal.get()).values).toEqual([])+ })+it('isolates a blackholed submission and continues flushing later telemetry', async () => {infrastructureTelemetryGapsTotal.reset()const fetchFn = vidiff --git a/host-context-controller/src/infrastructureTelemetryReporter.ts b/host-context-controller/src/infrastructureTelemetryReporter.tsindex e7cf0441d..47c687f88 100644--- a/host-context-controller/src/infrastructureTelemetryReporter.ts+++ b/host-context-controller/src/infrastructureTelemetryReporter.ts@@ -8,15 +8,28 @@ import {infrastructureTelemetryGapsTotal,infrastructureTelemetryRetriesTotal,} from './metrics'+import { throwForFailedSubmit } from './reporterHttpFailure'import { signInternalControlJwt } from './utils/internalControlSigner'export type HccHealthTransitionProjection = {sourceEventId: stringoccurredAt: string- hostLookupReference: { name: string; namespace: string; generation?: number }+ hostLookupReference: HostLookupReferencepayload: { transition: string; state: string }}+/**+ * `uid` pins the event to one Host object. A Host deleted and recreated with+ * the same name restarts at generation 1; without the uid its events would+ * reuse the previous object's identities (#691).+ */+export type HostLookupReference = {+ name: string+ namespace: string+ generation?: number+ uid?: string+}+export type HccInfrastructureTelemetryType =| 'health_transition'| 'lifecycle_transition'@@ -35,15 +48,11 @@ export type HccInfrastructureTelemetryPayload = {detail_ref?: stringattempt?: numbercount?: number- gfs_subject?: string- gfs_outcome?: 'minted' | 'rotated' | 'reused' | 'failed'- gfs_old_host_uid?: string- gfs_new_host_uid?: string}type HccInfrastructureTelemetryProjectionBase = {occurredAt: string- hostLookupReference: { name: string; namespace: string; generation?: number }+ hostLookupReference: HostLookupReferencepayload?: HccInfrastructureTelemetryPayload}@@ -67,6 +76,7 @@ export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProjeprojection.hostLookupReference.namespace,projection.hostLookupReference.name,projection.hostLookupReference.generation ?? 0,+ projection.hostLookupReference.uid ?? null,payload.reason_code ?? null,payload.error_class ?? null,payload.phase ?? null,@@ -78,12 +88,10 @@ export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProjepayload.detail_ref ?? null,payload.attempt ?? null,payload.count ?? null,- payload.gfs_subject ?? null,- payload.gfs_outcome ?? null,- payload.gfs_old_host_uid ?? null,- payload.gfs_new_host_uid ?? null,]- return `hcc-reconcile-outcome-v2:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`+ // v3: the tuple gained the Host uid (#691) and lost the gfs_* fields, which+ // control-api never accepted (#328). v2 identities are not produced again.+ return `hcc-reconcile-outcome-v3:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`}export interface InfrastructureTelemetryReporter {@@ -143,6 +151,17 @@ export class BoundedInfrastructureTelemetryReporter implements InfrastructureTelonEnqueued: projection =>infrastructureTelemetryEnqueuedTotal.inc({ telemetry_type: projection.telemetryType }),onAccepted: () => infrastructureTelemetryFlushesTotal.inc({ result: 'accepted' }),+ onTerminal: (projection, result) => {+ infrastructureTelemetryFlushesTotal.inc({ result })+ // A conflict means a row already exists for the key. A rejected event+ // is never stored, so it is a gap in the evidence.+ if (result === 'rejected') {+ infrastructureTelemetryGapsTotal.inc({+ telemetry_type: projection.telemetryType,+ reason: result,+ })+ }+ },onRetry: projection =>infrastructureTelemetryRetriesTotal.inc({ telemetry_type: projection.telemetryType }),onDrop: (projection, reason) => this.recordDrop(projection.telemetryType, reason),@@ -210,7 +229,7 @@ export class BoundedInfrastructureTelemetryReporter implements InfrastructureTelsignal: controller.signal,})- if (!response.ok) throw new Error(`telemetry submit failed with ${response.status}`)+ await throwForFailedSubmit(response, 'telemetry')} finally {clearTimeout(timeout)}diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 135108158..4be4dfa82 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -239,7 +239,7 @@ export const infrastructureTelemetryGapsTotal = counter({export const administrativeOutcomeReporterTotal = counter({name: 'clerum_hcc_administrative_outcome_reporter_total',- help: 'Administrative outcome reporter enqueue, flush, and drop outcomes.',+ help: 'Administrative outcome reporter results: enqueued, deduplicated, accepted, conflict, rejected, or a drop reason.',labelNames: ['result'] as const,})diff --git a/host-context-controller/src/reporterHttpFailure.test.ts b/host-context-controller/src/reporterHttpFailure.test.tsnew file mode 100644index 000000000..e2af52fc7--- /dev/null+++ b/host-context-controller/src/reporterHttpFailure.test.ts@@ -0,0 +1,113 @@+import { afterEach, describe, expect, it, vi } from 'vitest'+import { ReporterTerminalError } from './boundedOffPathReporter'+import { HostContextLogger } from './logger'+import { throwForFailedSubmit } from './reporterHttpFailure'++function response(status: number, readBody: () => Promise<unknown>) {+ const json = vi.fn(readBody)+ return { response: { ok: status < 300, status, json } as unknown as Response, json }+}++function named(name: string): Error {+ const error = new Error(`${name} while reading the body`)+ error.name = name+ return error+}++async function failure(status: number, readBody: () => Promise<unknown>) {+ const { response: res } = response(status, readBody)+ return throwForFailedSubmit(res, 'test event').then(+ () => undefined,+ (error: unknown) => error+ )+}++afterEach(() => vi.restoreAllMocks())++describe('throwForFailedSubmit', () => {+ it('returns for a 2xx without reading the body', async () => {+ const { response: res, json } = response(202, async () => ({}))++ await expect(throwForFailedSubmit(res, 'test event')).resolves.toBeUndefined()+ expect(json).not.toHaveBeenCalled()+ })++ it.each([+ [409, 'tracing_idempotency_conflict', 'conflict'],+ [400, 'unsafe_tracing_input', 'rejected'],+ [400, 'invalid_tracing_input', 'rejected'],+ ] as const)('treats %i %s as terminal %s', async (status, code, result) => {+ const error = await failure(status, async () => ({ code, error: 'ignored' }))++ expect(error).toBeInstanceOf(ReporterTerminalError)+ expect(error).toMatchObject({ result })+ })++ it.each([+ [400, 'tracing_idempotency_conflict'],+ [409, 'unsafe_tracing_input'],+ [409, 'invalid_tracing_input'],+ [403, 'tracing_idempotency_conflict'],+ ] as const)(+ 'keeps %i with %s retryable: the status is part of the pair',+ async (status, code) => {+ const error = await failure(status, async () => ({ code }))++ expect(error).toBeInstanceOf(Error)+ expect(error).not.toBeInstanceOf(ReporterTerminalError)+ expect((error as Error).message).toBe(`test event submit failed with ${status}`)+ }+ )++ it.each([+ ['an array', [{ code: 'tracing_idempotency_conflict' }]],+ ['null', null],+ ['a numeric code', { code: 409 }],+ ['a string', 'tracing_idempotency_conflict'],+ ])('keeps a 409 whose body is %s retryable', async (_label, body) => {+ const error = await failure(409, async () => body)++ expect(error).toBeInstanceOf(Error)+ expect(error).not.toBeInstanceOf(ReporterTerminalError)+ })++ it('does not read the body of a status that has no terminal pair', async () => {+ const { response: res, json } = response(500, async () => ({ code: 'unsafe_tracing_input' }))++ await expect(throwForFailedSubmit(res, 'test event')).rejects.toThrow(+ 'test event submit failed with 500'+ )+ expect(json).not.toHaveBeenCalled()+ })++ it('keeps the body out of the error message', async () => {+ const error = await failure(409, async () => ({+ code: 'tracing_idempotency_conflict',+ error: 'body-marker',+ }))++ // Liveness: the body was read, since the code made it terminal.+ expect(error).toBeInstanceOf(ReporterTerminalError)+ expect((error as Error).message).not.toContain('body-marker')+ })++ it.each([+ ['SyntaxError', 'not_json'],+ ['AbortError', 'aborted'],+ ['TypeError', 'read_failed'],+ ])('logs a %s during the body read as %s and stays retryable', async (name, reason) => {+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => {})++ const error = await failure(409, async () => {+ throw named(name)+ })++ expect(error).not.toBeInstanceOf(ReporterTerminalError)+ expect(warn).toHaveBeenCalledOnce()+ expect(warn).toHaveBeenCalledWith('reporter could not read the response code', {+ label: 'test event',+ status: 409,+ reason,+ })+ })+})diff --git a/host-context-controller/src/reporterHttpFailure.ts b/host-context-controller/src/reporterHttpFailure.tsnew file mode 100644index 000000000..52062ae02--- /dev/null+++ b/host-context-controller/src/reporterHttpFailure.ts@@ -0,0 +1,65 @@+import { ReporterTerminalError, type ReporterTerminalResult } from './boundedOffPathReporter'+import { hccLogger } from './logger'++const log = hccLogger.child({ module: 'reporter-http-failure' })++/**+ * control-api answers these (status, code) pairs deterministically: resending+ * the same event gets the same answer. `invalid_tracing_input` comes only from+ * request shape validation, so it is as final as `unsafe_tracing_input`. Every other failure (403 binding not yet+ * visible, 5xx, a 4xx without a code, a body that is not JSON) stays retryable.+ */+const TERMINAL_RESPONSES: ReadonlyArray<{+ status: number+ code: string+ result: ReporterTerminalResult+}> = [+ { status: 409, code: 'tracing_idempotency_conflict', result: 'conflict' },+ { status: 400, code: 'unsafe_tracing_input', result: 'rejected' },+ { status: 400, code: 'invalid_tracing_input', result: 'rejected' },+]++function unreadableBodyReason(error: unknown): 'not_json' | 'aborted' | 'read_failed' {+ if (error instanceof Error && error.name === 'SyntaxError') return 'not_json'+ if (error instanceof Error && error.name === 'AbortError') return 'aborted'+ return 'read_failed'+}++async function responseCode(response: Response, label: string): Promise<string | undefined> {+ let body: unknown+ try {+ body = await response.json()+ } catch (error) {+ // No code can be read, so the caller treats the response as retryable.+ // Only the status and the failure kind are logged, never the body.+ log.warn('reporter could not read the response code', {+ label,+ status: response.status,+ reason: unreadableBodyReason(error),+ })+ return undefined+ }+ if (typeof body !== 'object' || body === null) return undefined+ const code = (body as { code?: unknown }).code+ return typeof code === 'string' ? code : undefined+}++/**+ * Throws for a non-2xx submit response. Only the `code` field of the body is+ * read; the body is never logged or attached to the error.+ */+export async function throwForFailedSubmit(response: Response, label: string): Promise<void> {+ if (response.ok) return+ const candidates = TERMINAL_RESPONSES.filter(entry => entry.status === response.status)+ if (candidates.length > 0) {+ const code = await responseCode(response, label)+ const terminal = candidates.find(entry => entry.code === code)+ if (terminal) {+ throw new ReporterTerminalError(+ terminal.result,+ `${label} submit rejected with ${response.status} ${terminal.code}`+ )+ }+ }+ throw new Error(`${label} submit failed with ${response.status}`)+}diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 8e5f85fc6..b3e79f1ad 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1493,6 +1493,26 @@ describe('HostReconciler stateless lifecycle — status write idempotence', () =).toHaveLength(1)})+ it('binds the committed health transition to the Host uid (#691)', async () => {+ const infrastructureTelemetryReporter = createTelemetryReporterMock()+ const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })+ await reconciler.reconcile({ ...makeStatelessHost(), generation: 3 })++ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledTimes(1)+ expect(infrastructureTelemetryReporter.enqueueHealthTransition).toHaveBeenCalledWith(+ expect.objectContaining({+ hostLookupReference: {+ name: 'stateless-host',+ namespace: 'mcp-host',+ generation: 3,+ uid: 'stateless-host-uid',+ },+ payload: { transition: 'lifecycle:suspended', state: 'suspended' },+ })+ )+ })+it('skips the write when the observed status already matches', async () => {const infrastructureTelemetryReporter = createTelemetryReporterMock()const { reconciler, customApi } = createReconciler({ infrastructureTelemetryReporter })diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex 062ad5e12..7100fb1b9 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -1,6 +1,9 @@-import { describe, expect, it, vi } from 'vitest'+import { describe, expect, it, onTestFinished, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'-import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeReporter'+import {+ type AdministrativeOutcomeReporter,+ BoundedAdministrativeOutcomeReporter,+} from '../src/administrativeOutcomeReporter'import { mintHostGfsToken } from '../src/gfsHostBinding'import {DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES,@@ -271,16 +274,24 @@ describe('HostReconciler', () => {'clerum.io/gfs-token-host-generation': '1','clerum.io/runtime-token-rollout-required': 'true',})- expect(reporter.enqueue).toHaveBeenCalledWith(- expect.objectContaining({- telemetryType: 'reconcile_outcome',- payload: expect.objectContaining({- gfs_subject: 'host:1st:mcp-host/alpha-host',- gfs_old_host_uid: 'old-host-uid',- gfs_new_host_uid: 'new-host-uid',- gfs_outcome: 'rotated',- }),- })+ // GFS evidence travels as the allowlisted `transition` (#328); the recreated+ // object is identified by the uid in the lookup reference (#691).+ const recreationOutcome = vi+ .mocked(reporter.enqueue)+ .mock.calls.map(([projection]) => projection)+ .filter(projection => projection.telemetryType === 'reconcile_outcome')+ .at(-1)+ expect(recreationOutcome).toMatchObject({+ hostLookupReference: {+ name: 'alpha-host',+ namespace: 'mcp-host',+ generation: 1,+ uid: 'new-host-uid',+ },+ payload: { transition: 'gfs_token:rotated' },+ })+ expect(Object.keys(recreationOutcome!.payload!).filter(key => key.startsWith('gfs_'))).toEqual(+ [])expect(appsApi.replaceNamespacedDeployment.mock.calls.every(@@ -350,7 +361,12 @@ describe('HostReconciler', () => {expect(reporter.enqueue).toHaveBeenCalledWith(expect.objectContaining({telemetryType: 'reconcile_outcome',- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 7 },+ hostLookupReference: {+ name: 'alpha-host',+ namespace: 'mcp-host',+ generation: 7,+ uid: 'host-uid-1',+ },payload: expect.objectContaining({resource_class: 'Host',reason_code: 'ready',@@ -389,6 +405,210 @@ describe('HostReconciler', () => {)})+ it('sends an administrative outcome once across repeated reconcile passes (#327)', async () => {+ const fetchFn = vi.fn().mockResolvedValue({ ok: true }) as unknown as typeof fetch+ const administrativeOutcomeReporter = new BoundedAdministrativeOutcomeReporter({+ baseUrl: 'http://control-api.test:8090',+ signToken: () => 'signed',+ fetchFn,+ random: () => 0,+ })+ onTestFinished(() => administrativeOutcomeReporter.stop())+ const telemetry = createTelemetryReporterMock()+ const { reconciler } = createReconciler({+ administrativeOutcomeReporter,+ infrastructureTelemetryReporter: telemetry,+ })+ const host = makeHost({+ generation: 7,+ annotations: { 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111' },+ })++ for (let pass = 0; pass < 3; pass += 1) {+ await reconciler.reconcile(host)+ await new Promise(resolve => setTimeout(resolve, 0))+ }++ // Liveness: every pass re-observed the outcome.+ expect(+ vi+ .mocked(telemetry.enqueue)+ .mock.calls.filter(([projection]) => projection.telemetryType === 'reconcile_outcome')+ ).toHaveLength(3)+ expect(fetchFn).toHaveBeenCalledOnce()+ })++ it('does not turn the previous pass status into a succeeded outcome when a pass throws (#327)', async () => {+ const administrativeOutcomeReporter = {+ enqueueHostOutcome: vi.fn(),+ stop: vi.fn(async () => undefined),+ }+ const telemetry = createTelemetryReporterMock()+ const { reconciler, rbacApi } = createReconciler({+ administrativeOutcomeReporter,+ infrastructureTelemetryReporter: telemetry,+ })+ const annotations = {+ 'clerum.io/administrative-intent-id': '11111111-1111-4111-8111-111111111111',+ }+ await reconciler.reconcile(makeHost({ generation: 7, annotations }))+ administrativeOutcomeReporter.enqueueHostOutcome.mockClear()+ vi.mocked(telemetry.enqueue).mockClear()+ // Fails before this pass writes any status, so the status read afterwards+ // is still the previous pass's deployed/ready.+ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })++ await expect(reconciler.reconcile(makeHost({ generation: 8, annotations }))).rejects.toEqual({+ code: 500,+ })++ // Liveness: the failed pass reported its error and its reconcile outcome+ // while the previous pass's ready status was still in place.+ expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })+ const types = vi+ .mocked(telemetry.enqueue)+ .mock.calls.map(([projection]) => projection.telemetryType)+ expect(types).toContain('controller_error')+ // The telemetry reports the exception, not the stale ready status.+ const failedOutcome = vi+ .mocked(telemetry.enqueue)+ .mock.calls.map(([projection]) => projection)+ .find(projection => projection.telemetryType === 'reconcile_outcome')+ expect(failedOutcome?.payload).toEqual({+ resource_class: 'Host',+ reason_code: 'reconcile_exception',+ status: 'failed',+ })+ expect(administrativeOutcomeReporter.enqueueHostOutcome).toHaveBeenCalledWith(+ expect.objectContaining({+ outcome: 'failed',+ hostRef: expect.objectContaining({ generation: 8 }),+ })+ )+ expect(administrativeOutcomeReporter.enqueueHostOutcome).not.toHaveBeenCalledWith(+ expect.objectContaining({ outcome: 'succeeded' })+ )+ })++ describe('GFS token evidence on reconcile_outcome (#328)', () => {+ function reconcileOutcomes(reporter: InfrastructureTelemetryReporter) {+ return vi+ .mocked(reporter.enqueue)+ .mock.calls.map(([projection]) => projection)+ .filter(projection => projection.telemetryType === 'reconcile_outcome')+ }++ it('reports minted once and not on the next pass', async () => {+ const reporter = createTelemetryReporterMock()+ const { reconciler, rbacApi, coreApi } = createReconciler({+ infrastructureTelemetryReporter: reporter,+ })+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!+ coreApi.readNamespacedSecret.mockImplementation((request: { name?: string } = {}) =>+ request.name === 'host-alpha-host-mcp-host-runtime-tokens'+ ? Promise.reject({ code: 404 })+ : readSecret(request)+ )+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length++ await reconciler.reconcile(makeHost({ generation: 3 }))+ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })+ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toEqual({+ code: 500,+ })++ // Liveness: both passes emitted an outcome; only the first reached the mint.+ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)+ const [first, second] = reconcileOutcomes(reporter)+ expect(first?.payload).toMatchObject({ transition: 'gfs_token:minted' })+ expect(second?.payload).toMatchObject({ reason_code: 'reconcile_exception' })+ expect(second?.payload).not.toHaveProperty('transition')+ })++ it('reports reused when the runtime Secret is kept', async () => {+ const reporter = createTelemetryReporterMock()+ const { reconciler, coreApi } = createReconciler({+ infrastructureTelemetryReporter: reporter,+ })+ // Reuse needs a refresh token whose `exp` can be read.+ const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url')+ const refreshToken = [+ encode({ alg: 'none' }),+ encode({ exp: Math.floor(Date.now() / 1000) + 30 * 86_400 }),+ 'unsigned',+ ].join('.')+ const issue = vi.mocked(issueMcpHostRuntimeTokens).getMockImplementation()!+ vi.mocked(issueMcpHostRuntimeTokens).mockImplementationOnce(async (...args) => ({+ ...(await issue(...args)),+ refreshToken,+ }))+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length++ await reconciler.reconcile(makeHost({ generation: 3 }))+ const firstWrite = coreApi.replaceNamespacedSecret.mock.calls.find(+ ([request]) => request.name === 'host-alpha-host-mcp-host-runtime-tokens'+ )?.[0].body as k8s.V1Secret+ coreApi.readNamespacedSecret.mockImplementation(({ name }: { name?: string } = {}) => {+ if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)+ return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })+ })+ await reconciler.reconcile(makeHost({ generation: 3 }))++ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)+ expect(reconcileOutcomes(reporter).map(outcome => outcome.payload?.transition)).toEqual([+ 'gfs_token:rotated',+ 'gfs_token:reused',+ ])+ })++ it('reports failed on the exception outcome when the mint is rejected', async () => {+ const reporter = createTelemetryReporterMock()+ const { reconciler } = createReconciler({ infrastructureTelemetryReporter: reporter })+ const validationError = new Error('gfs host token subject mismatch')+ validationError.name = 'GfsHostTokenValidationError'+ vi.mocked(mintHostGfsToken).mockRejectedValueOnce(validationError)++ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toThrow(+ /subject mismatch/+ )++ expect(reconcileOutcomes(reporter).map(outcome => outcome.payload)).toEqual([+ {+ resource_class: 'Host',+ reason_code: 'reconcile_exception',+ status: 'failed',+ transition: 'gfs_token:failed',+ },+ ])+ })++ it('drops the evidence of a superseded pass', async () => {+ const reporter = createTelemetryReporterMock()+ const { reconciler, appsApi, rbacApi } = createReconciler({+ infrastructureTelemetryReporter: reporter,+ })+ const mintsBefore = vi.mocked(mintHostGfsToken).mock.calls.length+ const superseded = new Error('Host spec changed while the Deployment write was in flight')+ superseded.name = 'HostMutationSpecRevisionChangedError'+ appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(superseded)++ await expect(reconciler.reconcile(makeHost({ generation: 3 }))).rejects.toBe(superseded)+ // Liveness: the superseded pass minted, and emitted no outcome.+ expect(vi.mocked(mintHostGfsToken).mock.calls.length - mintsBefore).toBe(1)+ expect(reconcileOutcomes(reporter)).toHaveLength(0)++ rbacApi.readNamespacedRole.mockRejectedValueOnce({ code: 500 })+ await expect(reconciler.reconcile(makeHost({ generation: 4 }))).rejects.toEqual({+ code: 500,+ })++ const outcomes = reconcileOutcomes(reporter)+ expect(outcomes).toHaveLength(1)+ expect(outcomes[0]?.payload).toMatchObject({ reason_code: 'reconcile_exception' })+ expect(outcomes[0]?.payload).not.toHaveProperty('transition')+ })+ })+it('emits Host-backed controller_error and failed reconcile_outcome on reconcile failure', async () => {const reporter = createTelemetryReporterMock()const { reconciler, coreApi } = createReconciler({ infrastructureTelemetryReporter: reporter })@@ -411,7 +631,12 @@ describe('HostReconciler', () => {expect(reporter.enqueue).toHaveBeenCalledWith(expect.objectContaining({telemetryType: 'controller_error',- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 8 },+ hostLookupReference: {+ name: 'alpha-host',+ namespace: 'mcp-host',+ generation: 8,+ uid: 'host-uid-1',+ },payload: expect.objectContaining({resource_class: 'Host',reason_code: 'SecretNotFound',@@ -423,7 +648,12 @@ describe('HostReconciler', () => {expect(reporter.enqueue).toHaveBeenCalledWith(expect.objectContaining({telemetryType: 'reconcile_outcome',- hostLookupReference: { name: 'alpha-host', namespace: 'mcp-host', generation: 8 },+ hostLookupReference: {+ name: 'alpha-host',+ namespace: 'mcp-host',+ generation: 8,+ uid: 'host-uid-1',+ },payload: expect.objectContaining({resource_class: 'Host',reason_code: 'not_ready',