Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): skip no-op token restarts and name /ready 503 clauses

Alfredo Lopezd00ce2787e83
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 73e46030c..baa942c8c 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -412,6 +412,9 @@ jobs:
- name: Test WRC-to-HCC contracts
run: bash scripts/tests/test-wrc-hcc-contracts-gate.sh
+ - name: Test inter-service token HCC restart skip
+ run: bash scripts/tests/test-inter-service-tokens.sh
+
- name: Test WRC internal dependency NetworkPolicy contracts
run: bash scripts/tests/test-wrc-internal-dependency-networkpolicy-gate.sh
diff --git a/deploy/base/control-plane/host-context-controller.yaml b/deploy/base/control-plane/host-context-controller.yaml
index e02982465..d6c0fa1e5 100644
--- a/deploy/base/control-plane/host-context-controller.yaml
+++ b/deploy/base/control-plane/host-context-controller.yaml
@@ -36,10 +36,12 @@ spec:
# reconciliation finishes, so time-to-Ready scales with the McpServer/Context
# fleet — clerum-dev took 651s at 108 McpServers / 22 Contexts. The unset default
# (600s) governed the deploy because `kubectl rollout status` aborts the instant
- # the deadline is exceeded, and the clock starts at apply-inter-service-tokens.sh's
- # `rollout restart`, so the wait inherits only the remaining budget (600-160=440s
- # vs a 480s timeout on 2026-08-05: #202's timeout never bound, deploy failed 50s
- # short). 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the
+ # the deadline is exceeded. apply-inter-service-tokens.sh used to start that
+ # clock on every token apply via `rollout restart`, even when
+ # INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET was unchanged; no-op token deploys no
+ # longer Recreate this Deployment, so they no longer consume this budget.
+ # A real HMAC change (or FORCE_CONSUMER_RESTART) still restarts HCC — then
+ # 1200s covers the 900s wait plus a 300s head-start. Mitigation, not the
# fix — #205 decouples readiness from fleet convergence, after which this should
# come back down.
progressDeadlineSeconds: 1200
diff --git a/deploy/scripts/apply-inter-service-tokens.sh b/deploy/scripts/apply-inter-service-tokens.sh
index 8f368cfe1..cdab1ec1c 100755
--- a/deploy/scripts/apply-inter-service-tokens.sh
+++ b/deploy/scripts/apply-inter-service-tokens.sh
@@ -43,6 +43,8 @@ set -euo pipefail
# control-api Secret already has a value.
# INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET override WRC HMAC (default: preserve-or-generate)
# INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET override HCC HMAC (default: preserve-or-generate)
+# FORCE_CONSUMER_RESTART if true, restart every consumer including
+# HCC even when the HCC HMAC is unchanged
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
# shellcheck source=lib/clerum-minikube-context.sh
@@ -229,6 +231,17 @@ kctl -n channels patch secret workflow-approval-request-reader-credentials \
-p='[{"op":"remove","path":"/data/service-token"}]' >/dev/null 2>&1 || true
# --- 2. internal-control-jwt-secrets (control-plane) ---
+# Compare the HCC-consumed key only. The merge patch writes WRC + HCC together,
+# so a whole-Secret resourceVersion (or parsing `patched (no change)`) would
+# Recreate HCC on a WRC-only rotation. Values stay in shell locals — never logged.
+HCC_HMAC_BEFORE="$(read_secret_key control-plane internal-control-jwt-secrets INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET)"
+if [ "${FORCE_CONSUMER_RESTART:-}" = "true" ] || \
+ [ -z "$HCC_HMAC_BEFORE" ] || \
+ [ "$HCC_HMAC_BEFORE" != "$INTERNAL_CONTROL_HCC_HMAC" ]; then
+ HCC_SECRET_CHANGED=true
+else
+ HCC_SECRET_CHANGED=false
+fi
log "Patching Secret internal-control-jwt-secrets (control-plane)"
ensure_secret control-plane internal-control-jwt-secrets
INTERNAL_CONTROL_PATCH="$(jq -cn \
@@ -296,7 +309,9 @@ WA_READER_PATCH="$(jq -cn \
kctl -n channels patch secret workflow-approval-request-reader-credentials --type=merge -p "$WA_READER_PATCH"
# Roll the consumers so they pick up the fresh tokens on first deploy. Safe
-# no-ops if the Deployments do not yet exist.
+# no-ops if the Deployments do not yet exist. HCC is Recreate + replicas:1
+# and reloads HMAC only at process start, so skip that extra rollout when the
+# in-memory HCC key is unchanged (image/spec applies still roll HCC themselves).
for pair in "control-plane:control-api" \
"control-plane:workflow-recipes" \
"control-plane:host-context-controller" \
@@ -306,6 +321,11 @@ for pair in "control-plane:control-api" \
"channels:clerum-workflow-approval-request-reader"; do
ns="${pair%%:*}"
dep="${pair#*:}"
+ if [ "$ns" = "control-plane" ] && [ "$dep" = "host-context-controller" ] && \
+ [ "$HCC_SECRET_CHANGED" != "true" ]; then
+ log "Skipping rollout of $ns/$dep: hcc-hmac unchanged"
+ continue
+ fi
if kctl -n "$ns" get deploy "$dep" >/dev/null 2>&1; then
log "Rolling deployment $ns/$dep to pick up fresh Secret values"
kctl -n "$ns" rollout restart deploy "$dep" >/dev/null
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 50434a9be..470d48e14 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.232",
+ "version": "0.1.233",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.232",
+ "version": "0.1.233",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index d2d6ee4e2..bebc9f50a 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.232",
+ "version": "0.1.233",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 53518b889..6d9375fee 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -1439,6 +1439,57 @@ describe('McpServerWatcher startup', () => {
}
})
+ it('keeps getReadinessInventoryDetail aligned with isReadinessInventoryAuthoritative one gate at a time', () => {
+ const watcher = newContextAuthoritativeWatcher()
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).hostCacheSynced = true
+ markNetworkPolicyRevocationAuthoritative(watcher)
+
+ const expectAligned = (authoritative: boolean) => {
+ const detail = watcher.getReadinessInventoryDetail()
+ const fromDetail =
+ !detail.stopped &&
+ detail.mcpServerCacheSynced &&
+ detail.contextCacheSynced &&
+ detail.hostCacheSynced &&
+ detail.safetyInventoryCertified &&
+ detail.contextRevisionAligned &&
+ detail.serverRevisionAligned
+ expect(fromDetail).toBe(authoritative)
+ expect(watcher.isReadinessInventoryAuthoritative()).toBe(authoritative)
+ }
+
+ expectAligned(true)
+ ;(watcher as any).mcpServerCacheSynced = false
+ expectAligned(false)
+ expect(watcher.getReadinessInventoryDetail().mcpServerCacheSynced).toBe(false)
+ ;(watcher as any).mcpServerCacheSynced = true
+ ;(watcher as any).contextCacheSynced = false
+ expectAligned(false)
+ ;(watcher as any).contextCacheSynced = true
+ ;(watcher as any).hostCacheSynced = false
+ expectAligned(false)
+ ;(watcher as any).hostCacheSynced = true
+ try {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(false)
+ expectAligned(false)
+ expect(watcher.getReadinessInventoryDetail().safetyInventoryCertified).toBe(false)
+ } finally {
+ mocks.hasCertifiedSafetyInventory.mockReturnValue(true)
+ }
+ ;(watcher as any).networkPolicyRevocationContextRevision =
+ (watcher as any).contextDesiredRevision + 1
+ expectAligned(false)
+ expect(watcher.getReadinessInventoryDetail().contextRevisionAligned).toBe(false)
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ ;(watcher as any).networkPolicyRevocationServerRevision =
+ (watcher as any).mcpServerDesiredRevision + 1
+ expectAligned(false)
+ expect(watcher.getReadinessInventoryDetail().serverRevisionAligned).toBe(false)
+ markNetworkPolicyRevocationAuthoritative(watcher)
+ expectAligned(true)
+ })
+
it('ignores a late McpServer callback from a retired watch generation', async () => {
const callbacks: Array<(type: string, apiObj: any) => Promise<void>> = []
mocks.watch.mockImplementation(async (path, _options, callback) => {
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index e32fb5b70..9d16b4d91 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -1013,12 +1013,35 @@ export class McpServerWatcher implements McpServerProvider {
* by preserving replicas and disabling stateless suspension until recovery.
* SFS/GFS retain their established eventual-resync contract.
*/
+ getReadinessInventoryDetail(): {
+ stopped: boolean
+ mcpServerCacheSynced: boolean
+ contextCacheSynced: boolean
+ hostCacheSynced: boolean
+ safetyInventoryCertified: boolean
+ contextRevisionAligned: boolean
+ serverRevisionAligned: boolean
+ } {
+ return {
+ stopped: this.stopped,
+ mcpServerCacheSynced: this.mcpServerCacheSynced,
+ contextCacheSynced: this.contextCacheSynced,
+ hostCacheSynced: this.hostCacheSynced,
+ safetyInventoryCertified: this.netPolReconciler.hasCertifiedSafetyInventory(),
+ contextRevisionAligned:
+ this.networkPolicyRevocationContextRevision === this.contextDesiredRevision,
+ serverRevisionAligned:
+ this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision,
+ }
+ }
+
isReadinessInventoryAuthoritative(): boolean {
+ const detail = this.getReadinessInventoryDetail()
return (
- !this.stopped &&
- this.mcpServerCacheSynced &&
- this.contextCacheSynced &&
- this.hostCacheSynced &&
+ !detail.stopped &&
+ detail.mcpServerCacheSynced &&
+ detail.contextCacheSynced &&
+ detail.hostCacheSynced &&
// Certification is pinned to CONTENT identity (the desired-revision
// counters), not CHANNEL identity (the watch-generation counters). A
// "Premature close" reconnect re-LISTs the same inventory and bumps the
@@ -1031,9 +1054,9 @@ export class McpServerWatcher implements McpServerProvider {
// forces re-certification. Losing a delete fence bumps no revision, so
// hasCertifiedSafetyInventory() remains the fence for that one failure
// mode that no equality below can see.
- this.netPolReconciler.hasCertifiedSafetyInventory() &&
- this.networkPolicyRevocationContextRevision === this.contextDesiredRevision &&
- this.networkPolicyRevocationServerRevision === this.mcpServerDesiredRevision
+ detail.safetyInventoryCertified &&
+ detail.contextRevisionAligned &&
+ detail.serverRevisionAligned
)
}
diff --git a/host-context-controller/src/main.ts b/host-context-controller/src/main.ts
index d8cd848b5..a391b698e 100644
--- a/host-context-controller/src/main.ts
+++ b/host-context-controller/src/main.ts
@@ -21,7 +21,11 @@ import {
} from './k8sClient'
import { McpApiAuthenticator } from './mcpApiAuthentication'
import { McpAuthorizationService } from './mcpAuthorization'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
+import {
+ resolveHostAuthoritativeFn,
+ resolveProviderAuthoritativeFn,
+ resolveReadinessDetailFn,
+} from './readinessGate'
import { ContextMapperServer } from './server'
import { StatelessLifecycleTracker } from './statelessLifecycleTracker'
import {
@@ -145,6 +149,7 @@ async function main(): Promise<void> {
// server's fail-closed default would pin /api/v1/desktop/* at 503 forever in
// dev (R2-M1). See resolveHostAuthoritativeFn.
const hostAuthoritativeFn = resolveHostAuthoritativeFn(watcher)
+ const readinessDetailFn = resolveReadinessDetailFn(watcher)
// Stateless heartbeat consumption — mcp-host pods authenticate their
// heartbeats toward control-api's /mcp-host facade (control-api is the
@@ -186,7 +191,8 @@ async function main(): Promise<void> {
providerAuthoritativeFn,
hostAuthoritativeFn,
mcpAuthenticator,
- mcpAuthorization
+ mcpAuthorization,
+ readinessDetailFn
)
await server.start()
diff --git a/host-context-controller/src/readinessGate.test.ts b/host-context-controller/src/readinessGate.test.ts
index a9b522ab0..8d7a634ee 100644
--- a/host-context-controller/src/readinessGate.test.ts
+++ b/host-context-controller/src/readinessGate.test.ts
@@ -1,5 +1,26 @@
import { describe, expect, it } from 'vitest'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
+import {
+ type ReadinessInventoryDetail,
+ readinessReasonsFromDetail,
+ resolveHostAuthoritativeFn,
+ resolveProviderAuthoritativeFn,
+ resolveReadinessDetailFn,
+} from './readinessGate'
+
+function authoritativeDetail(
+ overrides: Partial<ReadinessInventoryDetail> = {}
+): ReadinessInventoryDetail {
+ return {
+ stopped: false,
+ mcpServerCacheSynced: true,
+ contextCacheSynced: true,
+ hostCacheSynced: true,
+ safetyInventoryCertified: true,
+ contextRevisionAligned: true,
+ serverRevisionAligned: true,
+ ...overrides,
+ }
+}
describe('resolveProviderAuthoritativeFn (dev-mode readiness gate — B1)', () => {
it('dev provider (no watcher) is unconditionally authoritative', () => {
@@ -51,3 +72,42 @@ describe('resolveHostAuthoritativeFn (dev-mode desktop gate — R2-M1)', () => {
expect(resolveHostAuthoritativeFn(watcher)()).toBe(true)
})
})
+
+describe('resolveReadinessDetailFn', () => {
+ it('omits detail when there is no watcher', () => {
+ expect(resolveReadinessDetailFn(null)).toBeUndefined()
+ })
+
+ it('delegates to the watcher inventory detail', () => {
+ const detail = authoritativeDetail({ mcpServerCacheSynced: false })
+ const watcher = { getReadinessInventoryDetail: () => detail }
+ expect(resolveReadinessDetailFn(watcher)?.()).toEqual(detail)
+ })
+})
+
+describe('readinessReasonsFromDetail', () => {
+ it('returns no reasons when every inventory clause is true', () => {
+ expect(readinessReasonsFromDetail(authoritativeDetail())).toEqual([])
+ })
+
+ it('maps each false clause to a closed reason key', () => {
+ expect(
+ readinessReasonsFromDetail(authoritativeDetail({ mcpServerCacheSynced: false }))
+ ).toEqual(['mcp_watch_unsynced'])
+ expect(readinessReasonsFromDetail(authoritativeDetail({ contextCacheSynced: false }))).toEqual([
+ 'context_watch_unsynced',
+ ])
+ expect(readinessReasonsFromDetail(authoritativeDetail({ hostCacheSynced: false }))).toEqual([
+ 'host_watch_unsynced',
+ ])
+ expect(
+ readinessReasonsFromDetail(authoritativeDetail({ safetyInventoryCertified: false }))
+ ).toEqual(['safety_pass_uncertified'])
+ expect(
+ readinessReasonsFromDetail(authoritativeDetail({ contextRevisionAligned: false }))
+ ).toEqual(['revocation_revision_mismatch'])
+ expect(
+ readinessReasonsFromDetail(authoritativeDetail({ serverRevisionAligned: false }))
+ ).toEqual(['revocation_revision_mismatch'])
+ })
+})
diff --git a/host-context-controller/src/readinessGate.ts b/host-context-controller/src/readinessGate.ts
index 88ff77ee6..6d0e9d64d 100644
--- a/host-context-controller/src/readinessGate.ts
+++ b/host-context-controller/src/readinessGate.ts
@@ -1,3 +1,46 @@
+/** Closed-gate booleans for /ready 503 `reasons`. No CR names, counts, or secrets. */
+export type ReadinessInventoryDetail = {
+ stopped: boolean
+ mcpServerCacheSynced: boolean
+ contextCacheSynced: boolean
+ hostCacheSynced: boolean
+ safetyInventoryCertified: boolean
+ contextRevisionAligned: boolean
+ serverRevisionAligned: boolean
+}
+
+export type ReadinessReason =
+ | 'mcp_watch_unsynced'
+ | 'context_watch_unsynced'
+ | 'host_watch_unsynced'
+ | 'safety_pass_uncertified'
+ | 'revocation_revision_mismatch'
+
+export function readinessReasonsFromDetail(detail: ReadinessInventoryDetail): ReadinessReason[] {
+ const reasons: ReadinessReason[] = []
+ if (!detail.mcpServerCacheSynced) reasons.push('mcp_watch_unsynced')
+ if (!detail.contextCacheSynced) reasons.push('context_watch_unsynced')
+ if (!detail.hostCacheSynced) reasons.push('host_watch_unsynced')
+ if (!detail.safetyInventoryCertified) reasons.push('safety_pass_uncertified')
+ if (!detail.contextRevisionAligned || !detail.serverRevisionAligned) {
+ reasons.push('revocation_revision_mismatch')
+ }
+ return reasons
+}
+
+/**
+ * Resolve per-clause readiness detail for structured /ready 503 bodies.
+ *
+ * Production (McpServerWatcher present): expose the live inventory booleans.
+ * Dev (no watcher): omit the detail fn so 503 bodies stay `{status, ready}`
+ * and do not invent watch/safety clauses that do not exist.
+ */
+export function resolveReadinessDetailFn(
+ watcher: { getReadinessInventoryDetail(): ReadinessInventoryDetail } | null
+): (() => ReadinessInventoryDetail) | undefined {
+ return watcher ? () => watcher.getReadinessInventoryDetail() : undefined
+}
+
/**
* Resolve the readiness-authority gate for the server from the provider.
*
diff --git a/host-context-controller/src/server.test.ts b/host-context-controller/src/server.test.ts
index 4202f3e30..95a89ada8 100644
--- a/host-context-controller/src/server.test.ts
+++ b/host-context-controller/src/server.test.ts
@@ -11,7 +11,11 @@ import {
McpAuthorizationService,
type McpAuthorizationStore,
} from './mcpAuthorization'
-import { resolveHostAuthoritativeFn, resolveProviderAuthoritativeFn } from './readinessGate'
+import {
+ type ReadinessInventoryDetail,
+ resolveHostAuthoritativeFn,
+ resolveProviderAuthoritativeFn,
+} from './readinessGate'
import { ContextMapperServer, McpHostApiRateLimiter } from './server'
import type { McpServerInfo } from './types'
@@ -636,3 +640,87 @@ describe('ContextMapperServer desktop status authority', () => {
})
})
})
+
+describe('ContextMapperServer /ready reasons', () => {
+ let server: ContextMapperServer | null = null
+
+ const authoritativeDetail = (): ReadinessInventoryDetail => ({
+ stopped: false,
+ mcpServerCacheSynced: true,
+ contextCacheSynced: true,
+ hostCacheSynced: true,
+ safetyInventoryCertified: true,
+ contextRevisionAligned: true,
+ serverRevisionAligned: true,
+ })
+
+ afterEach(async () => {
+ await server?.stop()
+ server = null
+ vi.restoreAllMocks()
+ })
+
+ it('adds closed-gate reasons on 503 and keeps the 200 body unchanged', async () => {
+ let detail = authoritativeDetail()
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
+ () =>
+ detail.mcpServerCacheSynced &&
+ detail.contextCacheSynced &&
+ detail.hostCacheSynced &&
+ detail.safetyInventoryCertified &&
+ detail.contextRevisionAligned &&
+ detail.serverRevisionAligned,
+ () => false,
+ undefined,
+ undefined,
+ () => detail
+ )
+ server.setReady(true)
+
+ let response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(200)
+ expect(JSON.parse(response.body)).toEqual({ status: 'ready', ready: true })
+ expect(JSON.parse(response.body)).not.toHaveProperty('reasons')
+
+ detail = { ...detail, mcpServerCacheSynced: false, safetyInventoryCertified: false }
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({
+ status: 'degraded',
+ ready: false,
+ reasons: ['mcp_watch_unsynced', 'safety_pass_uncertified'],
+ })
+ })
+
+ it('omits reasons when the detail fn is missing or throws', async () => {
+ server = new ContextMapperServer(new FakeProvider(), 0, undefined, undefined, () => false)
+ server.setReady(true)
+ let response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+
+ await server.stop()
+ server = new ContextMapperServer(
+ new FakeProvider(),
+ 0,
+ undefined,
+ undefined,
+ () => false,
+ () => false,
+ undefined,
+ undefined,
+ () => {
+ throw new Error('detail unavailable')
+ }
+ )
+ server.setReady(true)
+ response = await invoke(server, '/ready')
+ expect(response.statusCode).toBe(503)
+ expect(JSON.parse(response.body)).toEqual({ status: 'degraded', ready: false })
+ expect(JSON.parse(response.body)).not.toHaveProperty('reasons')
+ })
+})
diff --git a/host-context-controller/src/server.ts b/host-context-controller/src/server.ts
index 00b5df722..bcaa18d42 100644
--- a/host-context-controller/src/server.ts
+++ b/host-context-controller/src/server.ts
@@ -17,8 +17,15 @@ import {
toPublicMcpTransport,
} from './mcpAuthorization'
import { mcpHostApiRequestsTotal, registry } from './metrics'
+import {
+ type ReadinessInventoryDetail,
+ type ReadinessReason,
+ readinessReasonsFromDetail,
+} from './readinessGate'
import { McpServersResponse } from './types'
+const readinessLog = hccLogger.child({ module: 'readiness' })
+
const MCP_SELECTOR_RE = /^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$/
const MCP_REQUEST_BODY_LIMIT = 1024
const mcpApiLog = hccLogger.child({ module: 'mcp-host-api' })
@@ -75,7 +82,12 @@ export class McpHostApiRateLimiter {
type ReadinessState =
| { ready: true; status: 'ready' }
- | { ready: false; status: 'starting' | 'degraded'; message: string }
+ | {
+ ready: false
+ status: 'starting' | 'degraded'
+ message: string
+ reasons?: ReadinessReason[]
+ }
export class ContextMapperServer {
private server: http.Server | null = null
@@ -88,7 +100,9 @@ export class ContextMapperServer {
private mcpRateLimiter: McpHostApiRateLimiter
private providerAuthoritativeFn: () => boolean
private hostAuthoritativeFn: () => boolean
+ private readinessDetailFn: (() => ReadinessInventoryDetail) | undefined
private ready = false
+ private lastReadinessTransitionKey = ''
constructor(
provider: McpServerProvider,
@@ -104,7 +118,8 @@ export class ContextMapperServer {
// that omits it gets a 503 on desktop rather than a wrong 200 'inactive'.
hostAuthoritativeFn: () => boolean = () => false,
mcpAuthenticator?: McpApiAuthenticator,
- mcpAuthorization?: McpAuthorizationService
+ mcpAuthorization?: McpAuthorizationService,
+ readinessDetailFn?: () => ReadinessInventoryDetail
) {
this.provider = provider
this.port = port
@@ -115,6 +130,7 @@ export class ContextMapperServer {
this.mcpRateLimiter = new McpHostApiRateLimiter(config.mcpHostApiRateLimitPerMinute)
this.providerAuthoritativeFn = providerAuthoritativeFn
this.hostAuthoritativeFn = hostAuthoritativeFn
+ this.readinessDetailFn = readinessDetailFn
}
/**
@@ -137,28 +153,57 @@ export class ContextMapperServer {
this.ready = ready
}
+ private collectReadinessReasons(): ReadinessReason[] | undefined {
+ if (!this.readinessDetailFn) return undefined
+ try {
+ return readinessReasonsFromDetail(this.readinessDetailFn())
+ } catch (err) {
+ readinessLog.error('readiness detail check failed', { err })
+ return undefined
+ }
+ }
+
+ private emitReadinessTransition(state: ReadinessState): void {
+ const reasons = !state.ready ? (state.reasons ?? []) : []
+ const key = `${state.ready}:${state.status}:${reasons.join(',')}`
+ if (key === this.lastReadinessTransitionKey) return
+ this.lastReadinessTransitionKey = key
+ if (state.ready) {
+ readinessLog.info('readiness became ready')
+ return
+ }
+ readinessLog.warn('readiness not ready', { status: state.status, reasons })
+ }
+
private getReadinessState(): ReadinessState {
if (!this.ready) {
- return {
+ const starting: ReadinessState = {
ready: false,
status: 'starting',
message: 'Context mapper is still starting',
}
+ this.emitReadinessTransition(starting)
+ return starting
}
try {
if (this.providerAuthoritativeFn()) {
- return { ready: true, status: 'ready' }
+ const ready: ReadinessState = { ready: true, status: 'ready' }
+ this.emitReadinessTransition(ready)
+ return ready
}
} catch (err) {
console.error('[Server] Provider authority readiness check failed:', err)
}
- return {
+ const degraded: ReadinessState = {
ready: false,
status: 'degraded',
message: 'Context mapper provider inventory is not authoritative',
+ reasons: this.collectReadinessReasons(),
}
+ this.emitReadinessTransition(degraded)
+ return degraded
}
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
@@ -211,10 +256,14 @@ export class ContextMapperServer {
// Readiness check
if (req.method === 'GET' && url.pathname === '/ready') {
const readiness = this.getReadinessState()
- this.sendJson(res, readiness.ready ? 200 : 503, {
+ const body: { status: string; ready: boolean; reasons?: ReadinessReason[] } = {
status: readiness.status,
ready: readiness.ready,
- })
+ }
+ if (!readiness.ready && readiness.reasons && readiness.reasons.length > 0) {
+ body.reasons = readiness.reasons
+ }
+ this.sendJson(res, readiness.ready ? 200 : 503, body)
return
}
diff --git a/scripts/tests/test-inter-service-tokens.sh b/scripts/tests/test-inter-service-tokens.sh
index 824c2ecb1..bd2f8c187 100755
--- a/scripts/tests/test-inter-service-tokens.sh
+++ b/scripts/tests/test-inter-service-tokens.sh
@@ -6,6 +6,11 @@ TMP="${TMPDIR:-/tmp}/inter-service-tokens-test.$$"
mkdir -p "$TMP/bin"
trap 'rm -rf "$TMP"' EXIT
+HCC_OLD="aa11bb22cc33dd44ee55ff6677889900aa11bb22cc33dd44ee55ff6677889900"
+HCC_NEW="ff00ee11dd22cc33bb44aa5566778899ff00ee11dd22cc33bb44aa5566778899"
+WRC_OLD="1111111111111111111111111111111111111111111111111111111111111111"
+WRC_NEW="2222222222222222222222222222222222222222222222222222222222222222"
+
cat > "$TMP/bin/openssl" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
@@ -34,11 +39,33 @@ case "${args[0]:-}" in
case "${args[1]:-}" in
ns) exit 0 ;;
secret)
- # Existing Secret reads return empty in this harness so the script must
- # use minikube fallback or explicit env depending on CONTEXT.
+ jsonpath=""
+ for ((i=0; i<${#args[@]}; i++)); do
+ if [[ "${args[$i]}" == "-o" ]]; then
+ jsonpath="${args[$((i+1))]:-}"
+ elif [[ "${args[$i]}" == -ojsonpath=* ]]; then
+ jsonpath="${args[$i]#-ojsonpath=}"
+ elif [[ "${args[$i]}" == -ojson ]]; then
+ jsonpath="json"
+ fi
+ done
+ if [[ "$jsonpath" == "json" ]]; then
+ printf '%s\n' '{"data":{}}'
+ exit 0
+ fi
+ if [[ "$ns" == "control-plane" && "${args[2]:-}" == "internal-control-jwt-secrets" ]]; then
+ if [[ "$jsonpath" == *INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET* && -n "${KUBE_SECRET_HCC:-}" ]]; then
+ printf '%s' "$KUBE_SECRET_HCC" | base64 | tr -d '\n'
+ elif [[ "$jsonpath" == *INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET* && -n "${KUBE_SECRET_WRC:-}" ]]; then
+ printf '%s' "$KUBE_SECRET_WRC" | base64 | tr -d '\n'
+ fi
+ fi
exit 0
;;
deploy|deployment)
+ if [[ "${KUBE_DEPLOY_EXISTS:-}" == "1" ]]; then
+ exit 0
+ fi
exit 1
;;
esac
@@ -57,6 +84,9 @@ case "${args[0]:-}" in
exit 0
;;
rollout)
+ if [[ "${args[1]:-}" == "restart" && -n "${ROLLOUT_LOG:-}" ]]; then
+ printf '%s %s\n' "$ns" "${args[*]}" >> "$ROLLOUT_LOG"
+ fi
exit 0
;;
esac
@@ -65,10 +95,43 @@ SH
chmod +x "$TMP/bin/openssl" "$TMP/bin/kubectl"
+mkdir -p "$TMP/sibling/scripts/minikube"
+printf '%s\n' 'DEV_HMAC_SECRET="dev-member-registration-hmac-secret"' \
+ > "$TMP/sibling/scripts/minikube/deploy-evenfire-member-registration.sh"
+
+assert_no_secret_material() {
+ local log
+ for log in "$TMP/stdout" "$TMP/stderr"; do
+ if grep -E 'aa11bb22cc33dd44|ff00ee11dd22cc33|1111111111111111|2222222222222222|0123456789abcdef0123456789abcdef' "$log" >/dev/null; then
+ echo "secret material leaked into $log" >&2
+ cat "$log" >&2
+ exit 1
+ fi
+ done
+}
+
run_apply() {
local context="$1" capture="$2"
shift 2
- CAPTURE_FILE="$capture" PATH="$TMP/bin:$PATH" CONTEXT="$context" "$@" bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"
+ CAPTURE_FILE="$capture" PATH="$TMP/bin:$PATH" CONTEXT="$context" \
+ CLERUM_PROJECT_DIR="$TMP/sibling" "$@" \
+ bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"
+}
+
+run_hcc_apply() {
+ local rollout="$1"
+ shift
+ : > "$rollout"
+ CAPTURE_FILE="$TMP/hcc-capture.json" ROLLOUT_LOG="$rollout" KUBE_DEPLOY_EXISTS=1 \
+ PATH="$TMP/bin:$PATH" CONTEXT=gke-dev CLERUM_PROJECT_DIR="$TMP/sibling" "$@" \
+ bash "$ROOT/deploy/scripts/apply-inter-service-tokens.sh" >"$TMP/stdout" 2>"$TMP/stderr"
+}
+
+assert_other_consumers_restarted() {
+ local rollout="$1"
+ grep -q 'control-plane .*restart deploy control-api' "$rollout"
+ grep -q 'control-plane .*restart deploy workflow-recipes' "$rollout"
+ grep -q 'profiles .*restart deploy external-rest-api' "$rollout"
}
minikube_capture="$TMP/minikube-control-api-internal-tokens.json"
@@ -94,4 +157,71 @@ env_capture="$TMP/env-control-api-internal-tokens.json"
run_apply gke-dev "$env_capture" env CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac
jq -e '.stringData.CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET == "test-member-registration-hmac"' "$env_capture" >/dev/null
+unchanged_rollout="$TMP/rollout-unchanged.log"
+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \
+ run_hcc_apply "$unchanged_rollout" env \
+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \
+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \
+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"
+grep -q 'Skipping rollout of control-plane/host-context-controller: hcc-hmac unchanged' "$TMP/stderr"
+if grep -q 'host-context-controller' "$unchanged_rollout"; then
+ echo "unchanged HCC HMAC must not restart host-context-controller" >&2
+ cat "$unchanged_rollout" >&2
+ exit 1
+fi
+assert_other_consumers_restarted "$unchanged_rollout"
+assert_no_secret_material
+
+changed_rollout="$TMP/rollout-changed.log"
+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \
+ run_hcc_apply "$changed_rollout" env \
+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \
+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_NEW" \
+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"
+if grep -q 'hcc-hmac unchanged' "$TMP/stderr"; then
+ echo "changed HCC HMAC must not skip HCC restart" >&2
+ exit 1
+fi
+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"
+grep -q 'host-context-controller' "$changed_rollout"
+assert_no_secret_material
+
+wrc_only_rollout="$TMP/rollout-wrc-only.log"
+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \
+ run_hcc_apply "$wrc_only_rollout" env \
+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \
+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \
+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_NEW"
+grep -q 'Skipping rollout of control-plane/host-context-controller: hcc-hmac unchanged' "$TMP/stderr"
+if grep -q 'host-context-controller' "$wrc_only_rollout"; then
+ echo "WRC-only rotation must not restart host-context-controller" >&2
+ cat "$wrc_only_rollout" >&2
+ exit 1
+fi
+assert_other_consumers_restarted "$wrc_only_rollout"
+assert_no_secret_material
+
+empty_before_rollout="$TMP/rollout-empty-before.log"
+run_hcc_apply "$empty_before_rollout" env \
+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \
+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_NEW"
+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"
+grep -q 'host-context-controller' "$empty_before_rollout"
+assert_no_secret_material
+
+force_rollout="$TMP/rollout-force.log"
+KUBE_SECRET_HCC="$HCC_OLD" KUBE_SECRET_WRC="$WRC_OLD" \
+ run_hcc_apply "$force_rollout" env \
+ FORCE_CONSUMER_RESTART=true \
+ CONTROL_API_MEMBER_REGISTRATION_HMAC_SECRET=test-member-registration-hmac \
+ INTERNAL_CONTROL_JWT_HCC_HMAC_SECRET="$HCC_OLD" \
+ INTERNAL_CONTROL_JWT_WRC_HMAC_SECRET="$WRC_OLD"
+if grep -q 'hcc-hmac unchanged' "$TMP/stderr"; then
+ echo "FORCE_CONSUMER_RESTART must restart HCC even when HMAC is unchanged" >&2
+ exit 1
+fi
+grep -q 'Rolling deployment control-plane/host-context-controller' "$TMP/stderr"
+grep -q 'host-context-controller' "$force_rollout"
+assert_no_secret_material
+
echo "inter-service token patch tests passed"