Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): skip no-op token restarts and name /ready 503 clauses
Alfredo Lopezd00ce2787e83
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 73e46030c..baa942c8c 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -412,6 +412,9 @@ jobs:- name: Test WRC-to-HCC contractsrun: bash scripts/tests/test-wrc-hcc-contracts-gate.sh+ - name: Test inter-service token HCC restart skip+ run: bash scripts/tests/test-inter-service-tokens.sh+- name: Test WRC internal dependency NetworkPolicy contractsrun: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.shdiff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yamlindex e02982465..d6c0fa1e5 100644--- a/deploy/base/control-plane/host-context-controller.yaml+++ b/deploy/base/control-plane/host-context-controller.yaml@@ -36,10 +36,12 @@ spec:# reconciliation finishes, so time-to-Ready scales with the McpServer/Context# fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default# (600s) governed the deploy because `kubectl rollout status` aborts the instant- # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's- # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s- # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s- # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the+ # the deadline is exceeded. apply-inter-service-tokens.sh used to start that+ # clock on every token apply via `rollout restart`, even when+ # INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET was unchanged; no-op token deploys no+ # longer Recreate this Deployment, so they no longer consume this budget.+ # A real HMAC change (or FORCE_CONSUMER_RESTART) still restarts HCC — then+ # 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the# fix — #205 decouples readiness from fleet convergence, after which this should# come back down.progressDeadlineSeconds: 1200diff --git a/deploy/scripts/apply-inter-service-tokens.sh b/deploy/scripts/apply-inter-service-tokens.shindex 8f368cfe1..cdab1ec1c 100755--- a/deploy/scripts/apply-inter-service-tokens.sh+++ b/deploy/scripts/apply-inter-service-tokens.sh@@ -43,6 +43,8 @@ set -euo pipefail# control-api Secret already has a value.# INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET override WRC HMAC (default: preserve-or-generate)# INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET override HCC HMAC (default: preserve-or-generate)+# FORCE_CONSUMER_RESTART if true, restart every consumer including+# HCC even when the HCC HMAC is unchangedSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"# shellcheck source=lib/clerum-minikube-context.sh@@ -229,6 +231,17 @@ kctl -n channels patch secret workflow-approval-request-reader-credentials \-p='[{"op":"remove","path":"/data/service-token"}]' >/dev/null 2>&1 || true# --- 2. internal-control-jwt-secrets (control-plane) ---+# Compare the HCC-consumed key only. The merge patch writes WRC + HCC together,+# so a whole-Secret resourceVersion (or parsing `patched (no change)`) would+# Recreate HCC on a WRC-only rotation. Values stay in shell locals — never logged.+HCC_HMAC_BEFORE="$(read_secret_key control-plane internal-control-jwt-secrets INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET)"+if [ "${FORCE_CONSUMER_RESTART:-}" = "true" ] || \+ [ -z "$HCC_HMAC_BEFORE" ] || \+ [ "$HCC_HMAC_BEFORE" != "$INTERNAL_CONTROL_HCC_HMAC" ]; then+ HCC_SECRET_CHANGED=true+else+ HCC_SECRET_CHANGED=false+filog "Patching Secret internal-control-jwt-secrets (control-plane)"ensure_secret control-plane internal-control-jwt-secretsINTERNAL_CONTROL_PATCH="$(jq -cn \@@ -296,7 +309,9 @@ WA_READER_PATCH="$(jq -cn \kctl -n channels patch secret workflow-approval-request-reader-credentials --type=merge -p "$WA_READER_PATCH"# Roll the consumers so they pick up the fresh tokens on first deploy. Safe-# no-ops if the Deployments do not yet exist.+# no-ops if the Deployments do not yet exist. HCC is Recreate + replicas:1+# and reloads HMAC only at process start, so skip that extra rollout when the+# in-memory HCC key is unchanged (image/spec applies still roll HCC themselves).for pair in "control-plane:control-api" \"control-plane:workflow-recipes" \"control-plane:host-context-controller" \@@ -306,6 +321,11 @@ for pair in "control-plane:control-api" \"channels:clerum-workflow-approval-request-reader"; dons="${pair%%:*}"dep="${pair#*:}"+ if [ "$ns" = "control-plane" ] && [ "$dep" = "host-context-controller" ] && \+ [ "$HCC_SECRET_CHANGED" != "true" ]; then+ log "Skipping rollout of $ns/$dep: hcc-hmac unchanged"+ continue+ fiif kctl -n "$ns" get deploy "$dep" >/dev/null 2>&1; thenlog "Rolling deployment $ns/$dep to pick up fresh Secret values"kctl -n "$ns" rollout restart deploy "$dep" >/dev/nulldiff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 50434a9be..470d48e14 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.232",+ "version": "0.1.233","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.232",+ "version": "0.1.233","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex d2d6ee4e2..bebc9f50a 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.232",+ "version": "0.1.233","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.tsindex 53518b889..6d9375fee 100644--- a/host-context-controller/src/k8sClient.test.ts+++ b/host-context-controller/src/k8sClient.test.ts@@ -1439,6 +1439,57 @@ describe('McpServerWatcher startup', () => {}})+ it('keeps getReadinessInventoryDetail aligned with isReadinessInventoryAuthoritative one gate at a time', () => {+ const watcher = newContextAuthoritativeWatcher()+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).hostCacheSynced = true+ markNetworkPolicyRevocationAuthoritative(watcher)++ const expectAligned = (authoritative: boolean) => {+ const detail = watcher.getReadinessInventoryDetail()+ const fromDetail =+ !detail.stopped &&+ detail.mcpServerCacheSynced &&+ detail.contextCacheSynced &&+ detail.hostCacheSynced &&+ detail.safetyInventoryCertified &&+ detail.contextRevisionAligned &&+ detail.serverRevisionAligned+ expect(fromDetail).toBe(authoritative)+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(authoritative)+ }++ expectAligned(true)+ ;(watcher as any).mcpServerCacheSynced = false+ expectAligned(false)+ expect(watcher.getReadinessInventoryDetail().mcpServerCacheSynced).toBe(false)+ ;(watcher as any).mcpServerCacheSynced = true+ ;(watcher as any).contextCacheSynced = false+ expectAligned(false)+ ;(watcher as any).contextCacheSynced = true+ ;(watcher as any).hostCacheSynced = false+ expectAligned(false)+ ;(watcher as any).hostCacheSynced = true+ try {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)+ expectAligned(false)+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)+ } finally {+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)+ }+ ;(watcher as any).networkPolicyRevocationContextRevision =+ (watcher as any).contextDesiredRevision + 1+ expectAligned(false)+ expect(watcher.getReadinessInventoryDetail().contextRevisionAligned).toBe(false)+ markNetworkPolicyRevocationAuthoritative(watcher)+ ;(watcher as any).networkPolicyRevocationServerRevision =+ (watcher as any).mcpServerDesiredRevision + 1+ expectAligned(false)+ expect(watcher.getReadinessInventoryDetail().serverRevisionAligned).toBe(false)+ markNetworkPolicyRevocationAuthoritative(watcher)+ expectAligned(true)+ })+it('ignores a late McpServer callback from a retired watch generation', async () => {const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []mocks.watch.mockImplementation(async (path, _options, callback) => {diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.tsindex e32fb5b70..9d16b4d91 100644--- a/host-context-controller/src/k8sClient.ts+++ b/host-context-controller/src/k8sClient.ts@@ -1013,12 +1013,35 @@ export class McpServerWatcher implements McpServerProvider {* by preserving replicas and disabling stateless suspension until recovery.* SFS/GFS retain their established eventual-resync contract.*/+ getReadinessInventoryDetail(): {+ stopped: boolean+ mcpServerCacheSynced: boolean+ contextCacheSynced: boolean+ hostCacheSynced: boolean+ safetyInventoryCertified: boolean+ contextRevisionAligned: boolean+ serverRevisionAligned: boolean+ } {+ return {+ stopped: this.stopped,+ mcpServerCacheSynced: this.mcpServerCacheSynced,+ contextCacheSynced: this.contextCacheSynced,+ hostCacheSynced: this.hostCacheSynced,+ safetyInventoryCertified: this.netPolReconciler.hasCertifiedSafetyInventory(),+ contextRevisionAligned:+ this.networkPolicyRevocationContextRevision === this.contextDesiredRevision,+ serverRevisionAligned:+ this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision,+ }+ }+isReadinessInventoryAuthoritative(): boolean {+ const detail = this.getReadinessInventoryDetail()return (- !this.stopped &&- this.mcpServerCacheSynced &&- this.contextCacheSynced &&- this.hostCacheSynced &&+ !detail.stopped &&+ detail.mcpServerCacheSynced &&+ detail.contextCacheSynced &&+ detail.hostCacheSynced &&// Certification is pinned to CONTENT identity (the desired-revision// counters), not CHANNEL identity (the watch-generation counters). A// "Premature close" reconnect re-LISTs the same inventory and bumps the@@ -1031,9 +1054,9 @@ export class McpServerWatcher implements McpServerProvider {// forces re-certification. Losing a delete fence bumps no revision, so// hasCertifiedSafetyInventory() remains the fence for that one failure// mode that no equality below can see.- this.netPolReconciler.hasCertifiedSafetyInventory() &&- this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&- this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision+ detail.safetyInventoryCertified &&+ detail.contextRevisionAligned &&+ detail.serverRevisionAligned)}diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.tsindex d8cd848b5..a391b698e 100644--- a/host-context-controller/src/main.ts+++ b/host-context-controller/src/main.ts@@ -21,7 +21,11 @@ import {} from './k8sClient'import { McpApiAuthenticator } from './mcpApiAuthentication'import { McpAuthorizationService } from './mcpAuthorization'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'+import {+ resolveHostAuthoritativeFn,+ resolveProviderAuthoritativeFn,+ resolveReadinessDetailFn,+} from './readinessGate'import { ContextMapperServer } from './server'import { StatelessLifecycleTracker } from './statelessLifecycleTracker'import {@@ -145,6 +149,7 @@ async function main(): Promise<void> {// server's fail-closed default would pin /api/v1/desktop/* at 503 forever in// dev (R2-M1). See resolveHostAuthoritativeFn.const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)+ const readinessDetailFn = resolveReadinessDetailFn(watcher)// Stateless heartbeat consumption — mcp-host pods authenticate their// heartbeats toward control-api's /mcp-host facade (control-api is the@@ -186,7 +191,8 @@ async function main(): Promise<void> {providerAuthoritativeFn,hostAuthoritativeFn,mcpAuthenticator,- mcpAuthorization+ mcpAuthorization,+ readinessDetailFn)await server.start()diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.tsindex a9b522ab0..8d7a634ee 100644--- a/host-context-controller/src/readinessGate.test.ts+++ b/host-context-controller/src/readinessGate.test.ts@@ -1,5 +1,26 @@import { describe, expect, it } from 'vitest'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'+import {+ type ReadinessInventoryDetail,+ readinessReasonsFromDetail,+ resolveHostAuthoritativeFn,+ resolveProviderAuthoritativeFn,+ resolveReadinessDetailFn,+} from './readinessGate'++function authoritativeDetail(+ overrides: Partial<ReadinessInventoryDetail> = {}+): ReadinessInventoryDetail {+ return {+ stopped: false,+ mcpServerCacheSynced: true,+ contextCacheSynced: true,+ hostCacheSynced: true,+ safetyInventoryCertified: true,+ contextRevisionAligned: true,+ serverRevisionAligned: true,+ ...overrides,+ }+}describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {it('dev provider (no watcher) is unconditionally authoritative', () => {@@ -51,3 +72,42 @@ describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)})})++describe('resolveReadinessDetailFn', () => {+ it('omits detail when there is no watcher', () => {+ expect(resolveReadinessDetailFn(null)).toBeUndefined()+ })++ it('delegates to the watcher inventory detail', () => {+ const detail = authoritativeDetail({ mcpServerCacheSynced: false })+ const watcher = { getReadinessInventoryDetail: () => detail }+ expect(resolveReadinessDetailFn(watcher)?.()).toEqual(detail)+ })+})++describe('readinessReasonsFromDetail', () => {+ it('returns no reasons when every inventory clause is true', () => {+ expect(readinessReasonsFromDetail(authoritativeDetail())).toEqual([])+ })++ it('maps each false clause to a closed reason key', () => {+ expect(+ readinessReasonsFromDetail(authoritativeDetail({ mcpServerCacheSynced: false }))+ ).toEqual(['mcp_watch_unsynced'])+ expect(readinessReasonsFromDetail(authoritativeDetail({ contextCacheSynced: false }))).toEqual([+ 'context_watch_unsynced',+ ])+ expect(readinessReasonsFromDetail(authoritativeDetail({ hostCacheSynced: false }))).toEqual([+ 'host_watch_unsynced',+ ])+ expect(+ readinessReasonsFromDetail(authoritativeDetail({ safetyInventoryCertified: false }))+ ).toEqual(['safety_pass_uncertified'])+ expect(+ readinessReasonsFromDetail(authoritativeDetail({ contextRevisionAligned: false }))+ ).toEqual(['revocation_revision_mismatch'])+ expect(+ readinessReasonsFromDetail(authoritativeDetail({ serverRevisionAligned: false }))+ ).toEqual(['revocation_revision_mismatch'])+ })+})diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.tsindex 88ff77ee6..6d0e9d64d 100644--- a/host-context-controller/src/readinessGate.ts+++ b/host-context-controller/src/readinessGate.ts@@ -1,3 +1,46 @@+/** Closed-gate booleans for /ready 503 `reasons`. No CR names, counts, or secrets. */+export type ReadinessInventoryDetail = {+ stopped: boolean+ mcpServerCacheSynced: boolean+ contextCacheSynced: boolean+ hostCacheSynced: boolean+ safetyInventoryCertified: boolean+ contextRevisionAligned: boolean+ serverRevisionAligned: boolean+}++export type ReadinessReason =+ | 'mcp_watch_unsynced'+ | 'context_watch_unsynced'+ | 'host_watch_unsynced'+ | 'safety_pass_uncertified'+ | 'revocation_revision_mismatch'++export function readinessReasonsFromDetail(detail: ReadinessInventoryDetail): ReadinessReason[] {+ const reasons: ReadinessReason[] = []+ if (!detail.mcpServerCacheSynced) reasons.push('mcp_watch_unsynced')+ if (!detail.contextCacheSynced) reasons.push('context_watch_unsynced')+ if (!detail.hostCacheSynced) reasons.push('host_watch_unsynced')+ if (!detail.safetyInventoryCertified) reasons.push('safety_pass_uncertified')+ if (!detail.contextRevisionAligned || !detail.serverRevisionAligned) {+ reasons.push('revocation_revision_mismatch')+ }+ return reasons+}++/**+ * Resolve per-clause readiness detail for structured /ready 503 bodies.+ *+ * Production (McpServerWatcher present): expose the live inventory booleans.+ * Dev (no watcher): omit the detail fn so 503 bodies stay `{status, ready}`+ * and do not invent watch/safety clauses that do not exist.+ */+export function resolveReadinessDetailFn(+ watcher: { getReadinessInventoryDetail(): ReadinessInventoryDetail } | null+): (() => ReadinessInventoryDetail) | undefined {+ return watcher ? () => watcher.getReadinessInventoryDetail() : undefined+}+/*** Resolve the readiness-authority gate for the server from the provider.*diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.tsindex 4202f3e30..95a89ada8 100644--- a/host-context-controller/src/server.test.ts+++ b/host-context-controller/src/server.test.ts@@ -11,7 +11,11 @@ import {McpAuthorizationService,type McpAuthorizationStore,} from './mcpAuthorization'-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'+import {+ type ReadinessInventoryDetail,+ resolveHostAuthoritativeFn,+ resolveProviderAuthoritativeFn,+} from './readinessGate'import { ContextMapperServer, McpHostApiRateLimiter } from './server'import type { McpServerInfo } from './types'@@ -636,3 +640,87 @@ describe('ContextMapperServer desktop status authority', () => {})})})++describe('ContextMapperServer /ready reasons', () => {+ let server: ContextMapperServer | null = null++ const authoritativeDetail = (): ReadinessInventoryDetail => ({+ stopped: false,+ mcpServerCacheSynced: true,+ contextCacheSynced: true,+ hostCacheSynced: true,+ safetyInventoryCertified: true,+ contextRevisionAligned: true,+ serverRevisionAligned: true,+ })++ afterEach(async () => {+ await server?.stop()+ server = null+ vi.restoreAllMocks()+ })++ it('adds closed-gate reasons on 503 and keeps the 200 body unchanged', async () => {+ let detail = authoritativeDetail()+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,+ () =>+ detail.mcpServerCacheSynced &&+ detail.contextCacheSynced &&+ detail.hostCacheSynced &&+ detail.safetyInventoryCertified &&+ detail.contextRevisionAligned &&+ detail.serverRevisionAligned,+ () => false,+ undefined,+ undefined,+ () => detail+ )+ server.setReady(true)++ let response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(200)+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })+ expect(JSON.parse(response.body)).not.toHaveProperty('reasons')++ detail = { ...detail, mcpServerCacheSynced: false, safetyInventoryCertified: false }+ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({+ status: 'degraded',+ ready: false,+ reasons: ['mcp_watch_unsynced', 'safety_pass_uncertified'],+ })+ })++ it('omits reasons when the detail fn is missing or throws', async () => {+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => false)+ server.setReady(true)+ let response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })++ await server.stop()+ server = new ContextMapperServer(+ new FakeProvider(),+ 0,+ undefined,+ undefined,+ () => false,+ () => false,+ undefined,+ undefined,+ () => {+ throw new Error('detail unavailable')+ }+ )+ server.setReady(true)+ response = await invoke(server, '/ready')+ expect(response.statusCode).toBe(503)+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })+ expect(JSON.parse(response.body)).not.toHaveProperty('reasons')+ })+})diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.tsindex 00b5df722..bcaa18d42 100644--- a/host-context-controller/src/server.ts+++ b/host-context-controller/src/server.ts@@ -17,8 +17,15 @@ import {toPublicMcpTransport,} from './mcpAuthorization'import { mcpHostApiRequestsTotal, registry } from './metrics'+import {+ type ReadinessInventoryDetail,+ type ReadinessReason,+ readinessReasonsFromDetail,+} from './readinessGate'import { McpServersResponse } from './types'+const readinessLog = hccLogger.child({ module: 'readiness' })+const MCP_SELECTOR_RE = /^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/const MCP_REQUEST_BODY_LIMIT = 1024const mcpApiLog = hccLogger.child({ module: 'mcp-host-api' })@@ -75,7 +82,12 @@ export class McpHostApiRateLimiter {type ReadinessState =| { ready: true; status: 'ready' }- | { ready: false; status: 'starting' | 'degraded'; message: string }+ | {+ ready: false+ status: 'starting' | 'degraded'+ message: string+ reasons?: ReadinessReason[]+ }export class ContextMapperServer {private server: http.Server | null = null@@ -88,7 +100,9 @@ export class ContextMapperServer {private mcpRateLimiter: McpHostApiRateLimiterprivate providerAuthoritativeFn: () => booleanprivate hostAuthoritativeFn: () => boolean+ private readinessDetailFn: (() => ReadinessInventoryDetail) | undefinedprivate ready = false+ private lastReadinessTransitionKey = ''constructor(provider: McpServerProvider,@@ -104,7 +118,8 @@ export class ContextMapperServer {// that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.hostAuthoritativeFn: () => boolean = () => false,mcpAuthenticator?: McpApiAuthenticator,- mcpAuthorization?: McpAuthorizationService+ mcpAuthorization?: McpAuthorizationService,+ readinessDetailFn?: () => ReadinessInventoryDetail) {this.provider = providerthis.port = port@@ -115,6 +130,7 @@ export class ContextMapperServer {this.mcpRateLimiter = new McpHostApiRateLimiter(config.mcpHostApiRateLimitPerMinute)this.providerAuthoritativeFn = providerAuthoritativeFnthis.hostAuthoritativeFn = hostAuthoritativeFn+ this.readinessDetailFn = readinessDetailFn}/**@@ -137,28 +153,57 @@ export class ContextMapperServer {this.ready = ready}+ private collectReadinessReasons(): ReadinessReason[] | undefined {+ if (!this.readinessDetailFn) return undefined+ try {+ return readinessReasonsFromDetail(this.readinessDetailFn())+ } catch (err) {+ readinessLog.error('readiness detail check failed', { err })+ return undefined+ }+ }++ private emitReadinessTransition(state: ReadinessState): void {+ const reasons = !state.ready ? (state.reasons ?? []) : []+ const key = `${state.ready}:${state.status}:${reasons.join(',')}`+ if (key === this.lastReadinessTransitionKey) return+ this.lastReadinessTransitionKey = key+ if (state.ready) {+ readinessLog.info('readiness became ready')+ return+ }+ readinessLog.warn('readiness not ready', { status: state.status, reasons })+ }+private getReadinessState(): ReadinessState {if (!this.ready) {- return {+ const starting: ReadinessState = {ready: false,status: 'starting',message: 'Context mapper is still starting',}+ this.emitReadinessTransition(starting)+ return starting}try {if (this.providerAuthoritativeFn()) {- return { ready: true, status: 'ready' }+ const ready: ReadinessState = { ready: true, status: 'ready' }+ this.emitReadinessTransition(ready)+ return ready}} catch (err) {console.error('[Server] Provider authority readiness check failed:', err)}- return {+ const degraded: ReadinessState = {ready: false,status: 'degraded',message: 'Context mapper provider inventory is not authoritative',+ reasons: this.collectReadinessReasons(),}+ this.emitReadinessTransition(degraded)+ return degraded}private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {@@ -211,10 +256,14 @@ export class ContextMapperServer {// Readiness checkif (req.method === 'GET' && url.pathname === '/ready') {const readiness = this.getReadinessState()- this.sendJson(res, readiness.ready ? 200 : 503, {+ const body: { status: string; ready: boolean; reasons?: ReadinessReason[] } = {status: readiness.status,ready: readiness.ready,- })+ }+ if (!readiness.ready && readiness.reasons && readiness.reasons.length > 0) {+ body.reasons = readiness.reasons+ }+ this.sendJson(res, readiness.ready ? 200 : 503, body)return}diff --git a/scripts/tests/test-inter-service-tokens.sh b/scripts/tests/test-inter-service-tokens.shindex 824c2ecb1..bd2f8c187 100755--- a/scripts/tests/test-inter-service-tokens.sh+++ b/scripts/tests/test-inter-service-tokens.sh@@ -6,6 +6,11 @@ TMP="${TMPDIR:-/tmp}/inter-service-tokens-test.$$"mkdir -p "$TMP/bin"trap 'rm -rf "$TMP"' EXIT+HCC_OLD="aa11bb22cc33dd44ee55ff6677889900aa11bb22cc33dd44ee55ff6677889900"+HCC_NEW="ff00ee11dd22cc33bb44aa5566778899ff00ee11dd22cc33bb44aa5566778899"+WRC_OLD="1111111111111111111111111111111111111111111111111111111111111111"+WRC_NEW="2222222222222222222222222222222222222222222222222222222222222222"+cat > "$TMP/bin/openssl" <<'SH'#!/usr/bin/env bashset -euo pipefail@@ -34,11 +39,33 @@ case "${args[0]:-}" incase "${args[1]:-}" inns) exit 0 ;;secret)- # Existing Secret reads return empty in this harness so the script must- # use minikube fallback or explicit env depending on CONTEXT.+ jsonpath=""+ for ((i=0; i<${#args[@]}; i++)); do+ if [[ "${args[$i]}" == "-o" ]]; then+ jsonpath="${args[$((i+1))]:-}"+ elif [[ "${args[$i]}" == -ojsonpath=* ]]; then+ jsonpath="${args[$i]#-ojsonpath=}"+ elif [[ "${args[$i]}" == -ojson ]]; then+ jsonpath="json"+ fi+ done+ if [[ "$jsonpath" == "json" ]]; then+ printf '%s\n' '{"data":{}}'+ exit 0+ fi+ if [[ "$ns" == "control-plane" && "${args[2]:-}" == "internal-control-jwt-secrets" ]]; then+ if [[ "$jsonpath" == *INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET* && -n "${KUBE_SECRET_HCC:-}" ]]; then+ printf '%s' "$KUBE_SECRET_HCC" | base64 | tr -d '\n'+ elif [[ "$jsonpath" == *INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET* && -n "${KUBE_SECRET_WRC:-}" ]]; then+ printf '%s' "$KUBE_SECRET_WRC" | base64 | tr -d '\n'+ fi+ fiexit 0;;deploy|deployment)+ if [[ "${KUBE_DEPLOY_EXISTS:-}" == "1" ]]; then+ exit 0+ fiexit 1;;esac@@ -57,6 +84,9 @@ case "${args[0]:-}" inexit 0;;rollout)+ if [[ "${args[1]:-}" == "restart" && -n "${ROLLOUT_LOG:-}" ]]; then+ printf '%s %s\n' "$ns" "${args[*]}" >> "$ROLLOUT_LOG"+ fiexit 0;;esac@@ -65,10 +95,43 @@ SHchmod +x "$TMP/bin/openssl" "$TMP/bin/kubectl"+mkdir -p "$TMP/sibling/scripts/minikube"+printf '%s\n' 'DEV_HMAC_SECRET="dev-member-registration-hmac-secret"' \+ > "$TMP/sibling/scripts/minikube/deploy-evenfire-member-registration.sh"++assert_no_secret_material() {+ local log+ for log in "$TMP/stdout" "$TMP/stderr"; do+ if grep -E 'aa11bb22cc33dd44|ff00ee11dd22cc33|1111111111111111|2222222222222222|0123456789abcdef0123456789abcdef' "$log" >/dev/null; then+ echo "secret material leaked into $log" >&2+ cat "$log" >&2+ exit 1+ fi+ done+}+run_apply() {local context="$1" capture="$2"shift 2- CAPTURE_FILE="$capture" PATH="$TMP/bin:$PATH" CONTEXT="$context" "$@" bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"+ CAPTURE_FILE="$capture" PATH="$TMP/bin:$PATH" CONTEXT="$context" \+ CLERUM_PROJECT_DIR="$TMP/sibling" "$@" \+ bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"+}++run_hcc_apply() {+ local rollout="$1"+ shift+ : > "$rollout"+ CAPTURE_FILE="$TMP/hcc-capture.json" ROLLOUT_LOG="$rollout" KUBE_DEPLOY_EXISTS=1 \+ PATH="$TMP/bin:$PATH" CONTEXT=gke-dev CLERUM_PROJECT_DIR="$TMP/sibling" "$@" \+ bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"+}++assert_other_consumers_restarted() {+ local rollout="$1"+ grep -q 'control-plane .*restart deploy control-api' "$rollout"+ grep -q 'control-plane .*restart deploy workflow-recipes' "$rollout"+ grep -q 'profiles .*restart deploy external-rest-api' "$rollout"}minikube_capture="$TMP/minikube-control-api-internal-tokens.json"@@ -94,4 +157,71 @@ env_capture="$TMP/env-control-api-internal-tokens.json"run_apply gke-dev "$env_capture" env CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmacjq -e '.stringData.CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET == "test-member-registration-hmac"' "$env_capture" >/dev/null+unchanged_rollout="$TMP/rollout-unchanged.log"+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \+ run_hcc_apply "$unchanged_rollout" env \+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"+grep -q 'Skipping rollout of control-plane/host-context-controller: hcc-hmac unchanged' "$TMP/stderr"+if grep -q 'host-context-controller' "$unchanged_rollout"; then+ echo "unchanged HCC HMAC must not restart host-context-controller" >&2+ cat "$unchanged_rollout" >&2+ exit 1+fi+assert_other_consumers_restarted "$unchanged_rollout"+assert_no_secret_material++changed_rollout="$TMP/rollout-changed.log"+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \+ run_hcc_apply "$changed_rollout" env \+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_NEW" \+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"+if grep -q 'hcc-hmac unchanged' "$TMP/stderr"; then+ echo "changed HCC HMAC must not skip HCC restart" >&2+ exit 1+fi+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"+grep -q 'host-context-controller' "$changed_rollout"+assert_no_secret_material++wrc_only_rollout="$TMP/rollout-wrc-only.log"+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \+ run_hcc_apply "$wrc_only_rollout" env \+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_NEW"+grep -q 'Skipping rollout of control-plane/host-context-controller: hcc-hmac unchanged' "$TMP/stderr"+if grep -q 'host-context-controller' "$wrc_only_rollout"; then+ echo "WRC-only rotation must not restart host-context-controller" >&2+ cat "$wrc_only_rollout" >&2+ exit 1+fi+assert_other_consumers_restarted "$wrc_only_rollout"+assert_no_secret_material++empty_before_rollout="$TMP/rollout-empty-before.log"+run_hcc_apply "$empty_before_rollout" env \+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_NEW"+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"+grep -q 'host-context-controller' "$empty_before_rollout"+assert_no_secret_material++force_rollout="$TMP/rollout-force.log"+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \+ run_hcc_apply "$force_rollout" env \+ FORCE_CONSUMER_RESTART=true \+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"+if grep -q 'hcc-hmac unchanged' "$TMP/stderr"; then+ echo "FORCE_CONSUMER_RESTART must restart HCC even when HMAC is unchanged" >&2+ exit 1+fi+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"+grep -q 'host-context-controller' "$force_rollout"+assert_no_secret_material+echo "inter-service token patch tests passed"