Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): guard held-runtime credential renewal and bind wake bootstrap to the applied Deployment

Alfredo Lopez724f81878faa
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index d8e0de08a..a1411f78d 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.365",
+ "version": "0.1.366",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.365",
+ "version": "0.1.366",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 2889a4972..53372b194 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.365",
+ "version": "0.1.366",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index fd59d8a11..c4d0a6b98 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -720,6 +720,41 @@ export class HostReconciler {
* scope change mid-reconcile.
*/
private async frontsOAuthServer(host: HostCRD): Promise<boolean> {
+ const observation = await this.observeFrontsOAuthServer(host)
+ if (observation.observed) return observation.frontsOAuthServer
+ // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every
+ // attempt throws) still fails CLOSED here and can flip the scope off. That is
+ // correct — a sustained outage is a genuine "scope unknown" situation, not a
+ // blip — the retry only smooths over transient blips, never a real outage.
+ log.warn(
+ 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ contextRef: host.spec.contextRef,
+ attempts: observation.attempts,
+ err:
+ observation.error instanceof Error
+ ? observation.error.message
+ : String(observation.error),
+ }
+ )
+ return false
+ }
+
+ /**
+ * The live McpServer-backed OAuth observation behind `frontsOAuthServer`,
+ * reporting a read that failed after its retries instead of collapsing it to
+ * `false`. The CommunicationChannel-loss issuance path needs that distinction:
+ * it has no authoritative channel inventory, so it must not replace an
+ * unknown OAuth answer with a retained one that could re-grant a revoked scope.
+ */
+ private async observeFrontsOAuthServer(
+ host: HostCRD
+ ): Promise<
+ | { observed: true; frontsOAuthServer: boolean }
+ | { observed: false; attempts: number; error: unknown }
+ > {
// Bounded retry that absorbs a TRANSIENT apiserver blip without touching the
// authoritative-`false` path. Contract of the underlying resolver: a real
// scope change (the server genuinely stopped fronting an oauth mcp-server)
@@ -736,7 +771,7 @@ export class HostReconciler {
let lastErr: unknown
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
try {
- return await this.hostFrontsOAuthServerFn(host)
+ return { observed: true, frontsOAuthServer: await this.hostFrontsOAuthServerFn(host) }
} catch (err) {
lastErr = err
if (attempt < MAX_ATTEMPTS) {
@@ -745,21 +780,7 @@ export class HostReconciler {
}
}
}
- // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every
- // attempt throws) still fails CLOSED here and can flip the scope off. That is
- // correct — a sustained outage is a genuine "scope unknown" situation, not a
- // blip — the retry only smooths over transient blips, never a real outage.
- log.warn(
- 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
- {
- host: host.name,
- namespace: host.namespace,
- contextRef: host.spec.contextRef,
- attempts: MAX_ATTEMPTS,
- err: lastErr instanceof Error ? lastErr.message : String(lastErr),
- }
- )
- return false
+ return { observed: false, attempts: MAX_ATTEMPTS, error: lastErr }
}
/**
@@ -1566,6 +1587,18 @@ export class HostReconciler {
return (deployment?.status?.readyReplicas ?? 0) > 0
}
+ /**
+ * The binding covers the Deployment UID and its runtime-token-revision
+ * annotation only. It does not bind the rest of the pod template, so a
+ * template edit that keeps both values still matches.
+ *
+ * This is deliberate: there is no pod-template hash. The record and the
+ * Deployment belong to the same Host, and any principal that can edit the
+ * pod template can already read the runtime-token Secret, so a template hash
+ * would add no isolation. A hash would also have to be computed on the
+ * apiserver-defaulted object, which never equals the applied body, so it
+ * would fail to match and re-mint the credentials on every reconcile.
+ */
private static bootstrapBindingMatchesDeployment(
record: Pick<k8s.V1Secret, 'metadata'>,
deployment: k8s.V1Deployment | null
@@ -2012,22 +2045,39 @@ export class HostReconciler {
// codex projection into the mint-scope derive, the refresh decision, the
// scope hash and the stored annotation, so the minted token and the drift
// hash never diverge.
+ //
+ // The OAuth observation reads the McpServer cache, which does not depend
+ // on the CommunicationChannel cache, so it stays live during channel
+ // cache loss: an admin who removed the OAuth mcp-server while the channel
+ // watch was down must not have `oauth:user-token` re-granted from the
+ // retained annotation on a held wake. The retained value is used only
+ // when the live read failed after its retries and it cannot widen the
+ // grant (`false`); otherwise the mint is skipped (fail closed).
let observedFrontsOAuth: boolean
if (cacheSynced) {
observedFrontsOAuth = await this.frontsOAuthServer(host)
} else {
- if (retainedFrontsOAuth === null) {
+ const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)
+ if (liveFrontsOAuth.observed) {
+ observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer
+ } else if (retainedFrontsOAuth === false) {
+ observedFrontsOAuth = false
+ } else {
log.warn(
- 'skipping runtime token mint during channel cache loss without retained OAuth observation',
+ 'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
{
host: host.name,
namespace: host.namespace,
observation: 'frontsOAuthServer',
+ attempts: liveFrontsOAuth.attempts,
+ err:
+ liveFrontsOAuth.error instanceof Error
+ ? liveFrontsOAuth.error.message
+ : String(liveFrontsOAuth.error),
}
)
return null
}
- observedFrontsOAuth = retainedFrontsOAuth
}
const projection = this.projectCodexForHost(host)
const grokProjection = this.projectGrokForHost(host)
@@ -5064,7 +5114,6 @@ export class HostReconciler {
let pvcApplied = true
let serviceApplied = true
const resourceErrors: unknown[] = []
- const resourceFailureMessages = new Map<'Host PVC' | 'Host Service', string>()
const npFailures: string[] = []
const ensureIndependentResource = async (
kind: 'Host PVC' | 'Host Service',
@@ -5075,8 +5124,6 @@ export class HostReconciler {
} catch (error) {
if (isBenignSupersessionError(error)) throw error
resourceErrors.push(error)
- const message = error instanceof Error ? error.message : String(error)
- resourceFailureMessages.set(kind, `${kind}: ${message}`)
log.error('Failed to reconcile Host resource', {
host: host.name,
resource: kind,
@@ -5087,6 +5134,9 @@ export class HostReconciler {
}
const holdAppliedRuntime = async (wakeRequested: boolean): Promise<void> => {
const liveDeployment = await this.readHostDeploymentOrNull(host)
+ // A legacy Deployment without a runtime-token-revision annotation is not
+ // a preserved wake target, so it stays at zero replicas until channel
+ // authority returns and the normal path rewrites its template.
const preservedWakeTarget =
HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
(liveDeployment?.spec?.replicas ?? 1) === 0 &&
@@ -5129,7 +5179,9 @@ export class HostReconciler {
if (provisioningRequired) {
const provision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake: true,
- targetSuspended: lifecycle.effective.state === 'suspended',
+ // This mint exists only to scale the Host up from zero, so the
+ // target is never suspended.
+ targetSuspended: false,
preserveDeploymentTemplateOnWake: true,
})
if (!provision) {
@@ -5160,13 +5212,11 @@ export class HostReconciler {
(await this.checkDeploymentReady(host.name, host.namespace))
revalidateHostMutationBoundary()
const ready = deploymentReady && pvcApplied && serviceApplied && npFailures.length === 0
+ // PVC/Service read errors were already thrown before the hold; only
+ // non-throwing convergence failures reach this point.
const failures = [
- ...(!pvcApplied
- ? [resourceFailureMessages.get('Host PVC') ?? 'Host PVC did not converge']
- : []),
- ...(!serviceApplied
- ? [resourceFailureMessages.get('Host Service') ?? 'Host Service did not converge']
- : []),
+ ...(!pvcApplied ? ['Host PVC did not converge'] : []),
+ ...(!serviceApplied ? ['Host Service did not converge'] : []),
...npFailures,
]
this.advanceReconcileGeneration(host.name)
@@ -5188,6 +5238,17 @@ export class HostReconciler {
? 'CommunicationChannel inventory unavailable; preserving applied runtime'
: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
})
+ // Mirror the normal path: a runtime expected to run (including a held
+ // wake from zero) is polled until Ready; a degraded boundary is not,
+ // because the poll would overwrite the degraded verdict with "Running".
+ if (
+ applied &&
+ !deploymentReady &&
+ failures.length === 0 &&
+ (lifecycle.effective.state !== 'suspended' || allowScaleUp)
+ ) {
+ this.pollReadiness(host.name, host.namespace)
+ }
}
revalidateHostMutationBoundary()
@@ -5272,11 +5333,8 @@ export class HostReconciler {
}
revalidateHostMutationBoundary()
- if (lifecycle.effective.suspensionBlocked) {
- await holdAppliedRuntime(forceFreshForWake)
- return
- }
-
+ // PVC/Service errors abort before any credential or Deployment mutation,
+ // on the hold path as on the normal path.
if (resourceErrors.length === 1) throw resourceErrors[0]
if (resourceErrors.length > 1) {
throw new AggregateError(
@@ -5285,6 +5343,11 @@ export class HostReconciler {
)
}
+ if (lifecycle.effective.suspensionBlocked) {
+ await holdAppliedRuntime(forceFreshForWake)
+ return
+ }
+
// Bootstrap captures the scope contract used for issuance. The Deployment
// guard below compares that contract with the live channel cache after this
// potentially slow I/O and before every create/replace attempt.
diff --git a/host-context-controller/src/k8sClient.test.ts b/host-context-controller/src/k8sClient.test.ts
index 0419fbb3e..a7b2ce7e2 100644
--- a/host-context-controller/src/k8sClient.test.ts
+++ b/host-context-controller/src/k8sClient.test.ts
@@ -7480,7 +7480,7 @@ describe('McpServerWatcher.start ordering (#281 R6-bis)', () => {
expect(watcher.isCommunicationChannelCacheSynced()).toBe(false)
expect(errorSpy).toHaveBeenCalledWith(
'[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +
- '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',
+ '(B2 preserves channel-reader replicas, preserving durable Host lifecycle state):',
expect.any(Error)
)
diff --git a/host-context-controller/src/k8sClient.ts b/host-context-controller/src/k8sClient.ts
index d01c55f66..b758206e5 100644
--- a/host-context-controller/src/k8sClient.ts
+++ b/host-context-controller/src/k8sClient.ts
@@ -2828,7 +2828,7 @@ export class McpServerWatcher implements McpServerProvider {
} catch (error) {
console.error(
'[K8s] CommunicationChannel initial load failed; ccCacheSynced remains false ' +
- '(B2 preserves channel-reader replicas and holds stateless lifecycle active):',
+ '(B2 preserves channel-reader replicas, preserving durable Host lifecycle state):',
error
)
}
diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.ts
index f556dced4..479abe2d8 100644
--- a/host-context-controller/src/statelessLifecycleExecutor.ts
+++ b/host-context-controller/src/statelessLifecycleExecutor.ts
@@ -333,7 +333,7 @@ export class StatelessLifecycleExecutor {
}
if (!this.isCommunicationChannelCacheSynced()) {
- return this.holdActiveDuringChannelCacheRecovery(
+ return this.holdDurableStateDuringChannelCacheRecovery(
wakeHandledGeneration,
host.status?.lifecycle?.state ?? 'active'
)
@@ -370,7 +370,7 @@ export class StatelessLifecycleExecutor {
}
/** Unknown channel inventory requires preserving the applied Deployment. */
- private holdActiveDuringChannelCacheRecovery(
+ private holdDurableStateDuringChannelCacheRecovery(
wakeHandledGeneration: number,
state: HostLifecycleState = 'active'
): HostLifecycleAssessment {
@@ -411,7 +411,7 @@ export class StatelessLifecycleExecutor {
!this.isCommunicationChannelCacheSynced() &&
assessment.condition.reason === ACTIVE_COMMUNICATION_CHANNELS_REASON
) {
- return this.holdActiveDuringChannelCacheRecovery(
+ return this.holdDurableStateDuringChannelCacheRecovery(
assessment.lifecycle.wakeHandledGeneration,
assessment.effective.state
)
@@ -422,7 +422,7 @@ export class StatelessLifecycleExecutor {
const { reasons, messages } = this.communicationChannelPolicyRejection(hostName)
if (reasons.length === 0) {
if (!this.isCommunicationChannelCacheSynced() && !assessment.effective.suspensionBlocked) {
- return this.holdActiveDuringChannelCacheRecovery(
+ return this.holdDurableStateDuringChannelCacheRecovery(
assessment.lifecycle.wakeHandledGeneration,
assessment.effective.state
)
diff --git a/host-context-controller/src/statelessLifecycleTracker.ts b/host-context-controller/src/statelessLifecycleTracker.ts
index c25520012..087741c5a 100644
--- a/host-context-controller/src/statelessLifecycleTracker.ts
+++ b/host-context-controller/src/statelessLifecycleTracker.ts
@@ -296,6 +296,16 @@ export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
* because its persisted lastActivityTs predates the wake.
*/
private readonly wakeStartupGraceUntilByHost = new Map<string, number>()
+ /**
+ * Hosts whose durable state was already checked with a fresh read during the
+ * current suspension-blocked (CommunicationChannel authority loss) epoch. The
+ * cached Host can be stale about a `draining` write that landed just before
+ * the loss, so the first blocked beat bypasses the cached state once; later
+ * beats in the same epoch revert only on cached `draining` evidence instead of
+ * issuing a fresh GET and a skip log on every heartbeat of the outage. The
+ * entry is dropped on the first beat that is no longer blocked.
+ */
+ private readonly suspensionBlockedFreshCheckDone = new Set<string>()
/**
* Last FULLY successful heartbeat poll reported by the poller (AP-3 feed
* evidence). Undefined until the first report — the silent-emitter expiry
@@ -343,6 +353,7 @@ export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
this.lastBeatByHost.clear()
this.lastWakeHandledGenerationByHost.clear()
this.wakeStartupGraceUntilByHost.clear()
+ this.suspensionBlockedFreshCheckDone.clear()
}
/**
@@ -432,6 +443,9 @@ export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
return { drain: false }
}
const effective = this.reconciler.getEffectiveLifecycle(host)
+ if (!effective.suspensionBlocked) {
+ this.suspensionBlockedFreshCheckDone.delete(hostRef)
+ }
if (!effective.stateless) {
// Rejected/disabled stateless lifecycle: never drain. The reconciler's
// StatelessEnableRejected condition already names the reason.
@@ -440,7 +454,12 @@ export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
}
if (effective.suspensionBlocked) {
this.clearDrainGrace(hostRef)
- await this.cancelDrainOnEvidence(hostRef, host, true)
+ if (this.suspensionBlockedFreshCheckDone.has(hostRef)) {
+ await this.cancelDrainOnEvidence(hostRef, host)
+ } else if (await this.cancelDrainOnEvidence(hostRef, host, true)) {
+ // A failed fresh check is retried on the next blocked beat.
+ this.suspensionBlockedFreshCheckDone.add(hostRef)
+ }
return { drain: false }
}
this.noteWakeHandledGeneration(hostRef, host)
@@ -625,14 +644,16 @@ export class StatelessLifecycleTracker implements HeartbeatLifecycleTracker {
hostRef: string,
host: HostCRD,
bypassCachedState = false
- ): Promise<void> {
+ ): Promise<boolean> {
if (!bypassCachedState && (host.status?.lifecycle?.state ?? 'active') !== 'draining') {
- return
+ return true
}
try {
await this.reconciler.markHostActiveFromHeartbeat(host)
+ return true
} catch (err) {
console.error(`[StatelessLifecycleTracker] Cancel-drain revert failed for "${hostRef}":`, err)
+ return false
}
}
diff --git a/host-context-controller/test/hostReconciler.heartbeat.test.ts b/host-context-controller/test/hostReconciler.heartbeat.test.ts
index a1d9c7853..e3c9ff2c1 100644
--- a/host-context-controller/test/hostReconciler.heartbeat.test.ts
+++ b/host-context-controller/test/hostReconciler.heartbeat.test.ts
@@ -113,7 +113,7 @@ function makeStatelessHost(
}
}
-function createReconciler() {
+function createReconciler(options: { isCommunicationChannelCacheSynced?: () => boolean } = {}) {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const networkingApi = createMockNetworkingApi()
@@ -140,7 +140,7 @@ function createReconciler() {
// Heartbeat cases model the steady state after the watcher completed its
// CommunicationChannel initial list. Cache-startup fail-closed behavior
// is covered in hostReconciler.lifecycle.test.ts.
- isCommunicationChannelCacheSynced: () => true,
+ isCommunicationChannelCacheSynced: options.isCommunicationChannelCacheSynced ?? (() => true),
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi, customApi }
@@ -1528,6 +1528,34 @@ describe('HostReconciler.markHostActiveFromHeartbeat', () => {
await expect(reconciler.markHostActiveFromHeartbeat(host)).rejects.toThrow('api conflict')
expect(host.status?.lifecycle?.state).toBe('draining')
})
+
+ it('reverts a draining Host while channel authority loss blocks suspension', async () => {
+ // A suspension-blocked Host can still be draining from a decision made
+ // before the loss. Cancel-drain must stay available, otherwise the emitter
+ // remains fenced behind {drain:true} for the whole outage.
+ const { reconciler, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'draining', wakeHandledGeneration: 4 } },
+ })
+ customApi.getNamespacedCustomObject.mockResolvedValue(
+ freshHostRead({ state: 'draining', wakeHandledGeneration: 4 })
+ )
+ expect(reconciler.getEffectiveLifecycle(host)).toEqual({
+ stateless: true,
+ state: 'draining',
+ suspensionBlocked: true,
+ })
+
+ await reconciler.markHostActiveFromHeartbeat(host)
+
+ expect(customApi.getNamespacedCustomObject).toHaveBeenCalled()
+ const writes = lifecycleStatusWrites(customApi)
+ expect(writes).toHaveLength(1)
+ expect(writes[0].lifecycle).toEqual({ state: 'active', wakeHandledGeneration: 4 })
+ expect(host.status?.lifecycle?.state).toBe('active')
+ })
})
describe('HostReconciler heartbeat cores — fresh-read guard (cross-instance staleness)', () => {
// The Host watch callback builds a brand-new HostCRD per ADDED/MODIFIED
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index f6b2e51e7..ce3c386fa 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1034,7 +1034,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
it('never scales an unannotated legacy Deployment with an existing UID and resourceVersion', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
- const { reconciler, appsApi } = createReconciler()
+ const { reconciler, appsApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
const legacy = reconciler.buildDeployment(host)
delete legacy.metadata?.annotations?.['clerum.io/host-uid']
legacy.metadata = {
@@ -1043,44 +1045,118 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
resourceVersion: '73',
}
appsApi.readNamespacedDeployment.mockResolvedValue(legacy)
+ // Derive the held lifecycle from the Host itself so the fixture cannot
+ // contradict the durable suspended status.
+ const heldLifecycle = reconciler.getEffectiveLifecycle(host)
+ expect(heldLifecycle).toEqual({
+ stateless: true,
+ state: 'suspended',
+ suspensionBlocked: true,
+ })
await expect(
(reconciler as any).ensureDeployment(host, [], undefined, {
- stateless: true,
- state: 'active',
- suspensionBlocked: true,
+ ...heldLifecycle,
+ allowScaleUpDuringHold: true,
})
).rejects.toThrow('unverified Deployment')
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: host.name,
+ })
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
})
- it('converges the held runtime boundary and reports failures from PVC and policy reconciliation', async () => {
+ it('throws held-path PVC and Service errors before any credential or Deployment mutation', async () => {
const host = makeStatelessHost({ status: suspendedStatus() })
+ host.annotations = { 'clerum.io/wake-requested': '1' }
const { reconciler, appsApi, coreApi, customApi } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
- applied.status = { readyReplicas: 1 }
+ applied.spec!.replicas = 0
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, 0)
const live = persistHostDeployment(appsApi, host, applied)
const templateBefore = structuredClone(live().spec!.template)
- const replicasBefore = live().spec!.replicas
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+ const ingress = vi
+ .spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
+ .mockResolvedValue('up_to_date')
coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))
coreApi.readNamespacedService.mockRejectedValueOnce(new Error('Service read failed'))
+
+ const rejection = await reconciler.reconcile(host).then(
+ () => undefined,
+ (error: unknown) => error
+ )
+
+ expect(rejection).toBeInstanceOf(AggregateError)
+ expect((rejection as AggregateError).errors.map(error => (error as Error).message)).toEqual([
+ 'PVC read failed',
+ 'Service read failed',
+ ])
+ // Liveness witness: the pass reached the resource reads and the policy
+ // phase before rejecting.
+ expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: `${host.name}-workspace`,
+ })
+ expect(coreApi.readNamespacedService).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: host.name,
+ })
+ expect(ingress).toHaveBeenCalledWith(host)
+ expect(provision).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(0)
+ expect(live().spec!.template).toEqual(templateBefore)
+ })
+
+ it('rethrows a single held-path PVC error unwrapped and never enters the hold', async () => {
+ const host = makeStatelessHost({ status: suspendedStatus() })
+ host.annotations = { 'clerum.io/wake-requested': '1' }
+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const templateBefore = structuredClone(live().spec!.template)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
const ingress = vi
.spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
- .mockRejectedValueOnce(new Error('ingress policy failed'))
- const gfsEgress = vi
- .spyOn(reconciler as any, 'ensureMcpHostGfsEgressNetworkPolicy')
- .mockResolvedValue('up_to_date')
- const codexEgress = vi
- .spyOn(reconciler as any, 'reconcileMcpHostCodexProxyEgressNetworkPolicy')
.mockResolvedValue('up_to_date')
+ const ready = vi.spyOn(reconciler as any, 'checkDeploymentReady')
- await reconciler.reconcile(host)
+ coreApi.readNamespacedPersistentVolumeClaim.mockRejectedValueOnce(new Error('PVC read failed'))
+ const rejection = await reconciler.reconcile(host).then(
+ () => undefined,
+ (error: unknown) => error
+ )
+
+ expect(rejection).toBeInstanceOf(Error)
+ expect(rejection).not.toBeInstanceOf(AggregateError)
+ expect((rejection as Error).message).toBe('PVC read failed')
+ // Liveness witness: the pass read the PVC and Service and ran the policy
+ // phase, the last step before the throw. The hold path (which would mint
+ // credentials for this held wake and poll readiness) never ran.
expect(coreApi.readNamespacedPersistentVolumeClaim).toHaveBeenCalledWith({
namespace: host.namespace,
name: `${host.name}-workspace`,
@@ -1089,6 +1165,84 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
namespace: host.namespace,
name: host.name,
})
+ expect(ingress).toHaveBeenCalledWith(host)
+ expect(provision).not.toHaveBeenCalled()
+ expect(ready).not.toHaveBeenCalled()
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(0)
+ expect(live().spec!.template).toEqual(templateBefore)
+ })
+
+ it('waits for a verified applied runtime when a wake is requested and the live Deployment has no applied-runtime annotation', async () => {
+ const host = makeStatelessHost({ status: suspendedStatus() })
+ host.annotations = { 'clerum.io/wake-requested': '1' }
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ // A legacy Deployment: owned by the Host and at zero replicas, but its pod
+ // template carries no runtime-token-revision annotation.
+ const legacy = reconciler.buildDeployment(host)
+ legacy.spec!.replicas = 0
+ delete legacy.spec!.template!.metadata!.annotations?.['clerum.io/runtime-token-revision']
+ markPersistedRuntimeTrusted(legacy, host, 0)
+ const live = persistHostDeployment(appsApi, host, legacy)
+ const templateBefore = structuredClone(live().spec!.template)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
+
+ await reconciler.reconcile(host)
+
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: true,
+ ready: false,
+ message:
+ 'Waiting for a verified applied runtime before waking during CommunicationChannel inventory loss',
+ })
+ // Liveness witness: the held pass read the live Deployment before deciding.
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
+ namespace: host.namespace,
+ name: host.name,
+ })
+ expect(provision).not.toHaveBeenCalled()
+ expect(poll).not.toHaveBeenCalled()
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(0)
+ expect(live().spec!.template).toEqual(templateBefore)
+ })
+
+ it('converges the held runtime boundary and reports policy failures without polling', async () => {
+ // An active, not-Ready held runtime would be polled if the boundary were
+ // complete, so only the policy failure keeps the poll from running.
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady')
+ const templateBefore = structuredClone(live().spec!.template)
+ const replicasBefore = live().spec!.replicas
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness')
+
+ const ingress = vi
+ .spyOn(reconciler as any, 'ensureMcpHostIngressNetworkPolicy')
+ .mockRejectedValueOnce(new Error('ingress policy failed'))
+ const gfsEgress = vi
+ .spyOn(reconciler as any, 'ensureMcpHostGfsEgressNetworkPolicy')
+ .mockResolvedValue('up_to_date')
+ const codexEgress = vi
+ .spyOn(reconciler as any, 'reconcileMcpHostCodexProxyEgressNetworkPolicy')
+ .mockResolvedValue('up_to_date')
+
+ await reconciler.reconcile(host)
+
expect(ingress).toHaveBeenCalledOnce()
expect(ingress).toHaveBeenCalledWith(host)
expect(gfsEgress).toHaveBeenCalledOnce()
@@ -1098,14 +1252,49 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(reconciler.getStatus(host.name)).toMatchObject({
deployed: true,
ready: false,
- message: expect.stringContaining('Host PVC: PVC read failed'),
+ message: expect.stringContaining('mcp-host ingress: ingress policy failed'),
})
- expect(reconciler.getStatus(host.name)?.message).toContain('Host Service: Service read failed')
- expect(reconciler.getStatus(host.name)?.message).toContain(
- 'mcp-host ingress: ingress policy failed'
- )
+ expect(reconciler.getStatus(host.name)?.message).not.toContain('Host PVC')
+ expect(reconciler.getStatus(host.name)?.message).not.toContain('Host Service')
expect(reconciler.getStatus(host.name)?.message).not.toContain('GFS egress')
expect(reconciler.getStatus(host.name)?.message).not.toContain('Codex proxy egress')
+ // Liveness witness: the hold reached its readiness check and found the
+ // runtime not Ready, the state in which a complete boundary is polled.
+ expect(readySpy).toHaveBeenCalledWith(host.name, host.namespace)
+ await expect(readySpy.mock.results[0].value).resolves.toBe(false)
+ expect(poll).not.toHaveBeenCalled()
+ })
+
+ it('polls readiness after a held wake scales the Host up from zero', async () => {
+ const host = makeStatelessHost({ status: suspendedStatus() })
+ host.annotations = { 'clerum.io/wake-requested': '1' }
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
+ runtimeTokenProvision(host)
+ )
+ const poll = vi.spyOn(reconciler as any, 'pollReadiness').mockImplementation(() => undefined)
+
+ await reconciler.reconcile(host)
+
+ expect(live().spec!.replicas).toBe(1)
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: true,
+ ready: false,
+ message: 'CommunicationChannel inventory unavailable; preserved applied runtime is not Ready',
+ })
+ expect(poll).toHaveBeenCalledOnce()
+ expect(poll).toHaveBeenCalledWith(host.name, host.namespace)
})
it('does not report a Deployment applied when create conflict is followed by a missing read', async () => {
@@ -1150,6 +1339,49 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
)
})
+ it('fails closed when held credential renewal cannot verify the runtime identity', async () => {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ markPersistedRuntimeTrusted(applied, host)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const templateBefore = structuredClone(live().spec!.template)
+ // The record was minted for a previous Host incarnation, so the real
+ // refreshGfsOnly provision refuses to renew it and returns null.
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ runtimeCredentialRecord(host, { hostUid: 'prior-host-uid' }) as any
+ )
+ const runtimeSecretName = `host-${host.name}-mcp-host-runtime-tokens`
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+
+ await reconciler.reconcile(host)
+
+ // Liveness witness: the held renewal read the runtime credential record.
+ expect(coreApi.readNamespacedSecret).toHaveBeenCalledWith({
+ name: runtimeSecretName,
+ namespace: host.namespace,
+ })
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: true,
+ ready: false,
+ message: 'Held runtime credential renewal is not safely available',
+ })
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
+ expect(
+ coreApi.replaceNamespacedSecret.mock.calls.filter(
+ ([request]) => (request as { name?: string }).name === runtimeSecretName
+ )
+ ).toHaveLength(0)
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(1)
+ expect(live().spec!.template).toEqual(templateBefore)
+ })
+
it('keeps a suspended Host at zero replicas during cache loss without a pending wake', async () => {
const host = makeStatelessHost({ status: suspendedStatus(3) })
const { reconciler, appsApi, customApi } = createReconciler({
@@ -1342,36 +1574,160 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
)
})
- it('uses only retained scope observations while channel authority is unavailable', async () => {
+ it('uses the retained channel observation but a live OAuth observation while channel authority is unavailable', async () => {
const host = makeStatelessHost()
+ const countChannels = vi.fn(() => 0)
const { reconciler, coreApi } = createReconciler({
- countCommunicationChannels: () => 0,
+ countCommunicationChannels: countChannels,
isCommunicationChannelCacheSynced: () => false,
})
coreApi.readNamespacedSecret.mockResolvedValue(
runtimeCredentialRecord(host, {
- frontsOAuthServer: 'true',
+ frontsOAuthServer: 'false',
hasChannelIngress: 'true',
})
)
- const oauthProbe = vi
- .spyOn(reconciler as any, 'frontsOAuthServer')
- .mockImplementation(async () => {
- throw new Error('live lookup must not run')
- })
+ const oauthResolver = vi.fn(async () => true)
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
expect(provision).not.toBeNull()
- expect(oauthProbe).not.toHaveBeenCalled()
+ expect(oauthResolver).toHaveBeenCalledWith(host)
+ expect(countChannels).not.toHaveBeenCalled()
expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).toContain(
OAUTH_USER_TOKEN_SCOPE
)
const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
- expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(
- 'true'
+ expect(write.metadata?.annotations).toMatchObject({
+ 'clerum.io/runtime-token-fronts-oauth-server': 'true',
+ 'clerum.io/runtime-token-has-channel-ingress': 'true',
+ })
+ })
+
+ describe('held wake OAuth scope during channel cache loss', () => {
+ function heldZeroReplicaWake(options: {
+ cacheSynced: boolean
+ retainedFrontsOAuth: 'true' | 'false'
+ resolveFrontsOAuth: () => Promise<boolean>
+ }) {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => options.cacheSynced,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const runtimeRecord = runtimeCredentialRecord(host, {
+ frontsOAuthServer: options.retainedFrontsOAuth,
+ })
+ const runtimeRecordReads: string[] = []
+ coreApi.readNamespacedSecret.mockImplementation(({ name }) => {
+ if (name?.includes('runtime-tokens')) {
+ runtimeRecordReads.push(name)
+ return Promise.resolve(runtimeRecord)
+ }
+ return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} } as any)
+ })
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ const oauthResolver = vi.fn(options.resolveFrontsOAuth)
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+ return { host, reconciler, coreApi, live, oauthResolver, runtimeRecordReads }
+ }
+
+ function runtimeTokenWrite(coreApi: ReturnType<typeof createMockCoreApi>): k8s.V1Secret {
+ const write = coreApi.replaceNamespacedSecret.mock.calls.find(([request]) =>
+ request.name?.includes('runtime-tokens')
+ )?.[0].body as k8s.V1Secret | undefined
+ if (!write) throw new Error('No runtime token Secret write recorded')
+ return write
+ }
+
+ it.each([false, true])(
+ 'mints without oauth:user-token when the OAuth mcp-server was removed (cache synced: %s)',
+ async cacheSynced => {
+ const { host, reconciler, coreApi, live, oauthResolver } = heldZeroReplicaWake({
+ cacheSynced,
+ retainedFrontsOAuth: 'true',
+ resolveFrontsOAuth: async () => false,
+ })
+
+ await reconciler.reconcile(host)
+
+ expect(oauthResolver).toHaveBeenCalledWith(host)
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).toHaveBeenCalledOnce()
+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls[0]?.[2]).not.toContain(
+ OAUTH_USER_TOKEN_SCOPE
+ )
+ expect(
+ runtimeTokenWrite(coreApi).metadata?.annotations?.[
+ 'clerum.io/runtime-token-fronts-oauth-server'
+ ]
+ ).toBe('false')
+ expect(live().spec!.replicas).toBe(1)
+ }
)
+
+ it('skips a held wake mint when the OAuth read fails and the retained scope would grant it', async () => {
+ const { host, reconciler, coreApi, live, oauthResolver, runtimeRecordReads } =
+ heldZeroReplicaWake({
+ cacheSynced: false,
+ retainedFrontsOAuth: 'true',
+ resolveFrontsOAuth: async () => {
+ throw new Error('McpServer read unavailable')
+ },
+ })
+
+ await reconciler.reconcile(host)
+
+ expect(runtimeRecordReads.length).toBeGreaterThan(0)
+ expect(oauthResolver).toHaveBeenCalledTimes(3)
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
+ expect(
+ coreApi.replaceNamespacedSecret.mock.calls.filter(([request]) =>
+ request.name?.includes('runtime-tokens')
+ )
+ ).toHaveLength(0)
+ expect(live().spec!.replicas).toBe(0)
+ expect(reconciler.getStatus(host.name)).toMatchObject({
+ deployed: true,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ })
+
+ it('uses a retained non-granting OAuth observation when the OAuth read fails', async () => {
+ const { host, reconciler, coreApi, live, oauthResolver } = heldZeroReplicaWake({
+ cacheSynced: false,
+ retainedFrontsOAuth: 'false',
+ resolveFrontsOAuth: async () => {
+ throw new Error('McpServer read unavailable')
+ },
+ })
+
+ await reconciler.reconcile(host)
+
+ expect(oauthResolver).toHaveBeenCalledTimes(3)
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).toHaveBeenCalledOnce()
+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls[0]?.[2]).not.toContain(
+ OAUTH_USER_TOKEN_SCOPE
+ )
+ expect(
+ runtimeTokenWrite(coreApi).metadata?.annotations?.[
+ 'clerum.io/runtime-token-fronts-oauth-server'
+ ]
+ ).toBe('false')
+ expect(live().spec!.replicas).toBe(1)
+ })
})
it('does not renew held-runtime GFS before its refresh window', async () => {
@@ -1490,6 +1846,242 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
})
+ describe('held-runtime GFS renewal identity guards', () => {
+ const dueForRenewal = { gfsRefreshBefore: '2000-01-01T00:00:00.000Z' }
+
+ /** Runs one refreshGfsOnly provision against a due record and a Ready Deployment. */
+ async function renewHeldGfs(
+ scenario: {
+ host?: HostCRD
+ record?: (host: HostCRD) => k8s.V1Secret
+ deployment?: (reconciler: HostReconciler, host: HostCRD) => k8s.V1Deployment | null
+ } = {}
+ ) {
+ const host = scenario.host ?? makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ scenario.record ? scenario.record(host) : runtimeCredentialRecord(host, dueForRenewal)
+ )
+ const deployment = scenario.deployment
+ ? scenario.deployment(reconciler, host)
+ : trustedRuntimeDeployment(reconciler, host)
+ if (deployment) {
+ appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
+ } else {
+ appsApi.readNamespacedDeployment.mockRejectedValue({ code: 404 })
+ }
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ return {
+ provision,
+ mints: vi.mocked(mintHostGfsToken).mock.calls.length,
+ recordReads: coreApi.readNamespacedSecret.mock.calls.length,
+ deploymentReads: appsApi.readNamespacedDeployment.mock.calls.length,
+ recordWrites: coreApi.replaceNamespacedSecret.mock.calls.length,
+ }
+ }
+
+ /** The unmodified scenario mints, so every refusal below is a real refusal. */
+ async function expectControlRenews() {
+ const control = await renewHeldGfs()
+ expect(control.provision).not.toBeNull()
+ expect(control.mints).toBe(1)
+ expect(control.recordWrites).toBe(1)
+ }
+
+ async function expectRefusedAfterReads(scenario: Parameters<typeof renewHeldGfs>[0]) {
+ await expectControlRenews()
+ const refused = await renewHeldGfs(scenario)
+ // Liveness witness: the credential record and the Deployment were both read
+ // before the guard refused, so the guarded path was entered.
+ expect(refused.recordReads).toBeGreaterThan(0)
+ expect(refused.deploymentReads).toBeGreaterThan(0)
+ expect(refused.provision).toBeNull()
+ expect(refused.mints).toBe(0)
+ expect(refused.recordWrites).toBe(0)
+ }
+
+ it('refuses to renew an incomplete credential record', async () => {
+ await expectRefusedAfterReads({
+ record: host => {
+ const record = runtimeCredentialRecord(host, dueForRenewal)
+ delete record.data![MCP_HOST_GFS_TOKEN_SECRET_KEY]
+ return record
+ },
+ })
+ })
+
+ it.each([
+ [
+ 'managed by another controller',
+ (host: HostCRD) =>
+ runtimeCredentialRecord(host, { ...dueForRenewal, managedByHost: false }),
+ ],
+ [
+ 'labelled for another Host',
+ (host: HostCRD) => {
+ const record = runtimeCredentialRecord(host, dueForRenewal)
+ record.metadata!.labels!['clerum.io/host'] = 'other-host'
+ return record
+ },
+ ],
+ ])('refuses to renew a credential record %s', async (_label, record) => {
+ await expectRefusedAfterReads({ record })
+ })
+
+ it.each<[string, (deployment: k8s.V1Deployment) => void]>([
+ [
+ 'labelled for another Host',
+ deployment => {
+ deployment.metadata!.labels!['clerum.io/host'] = 'other-host'
+ },
+ ],
+ [
+ 'managed by another controller',
+ deployment => {
+ deployment.metadata!.labels!['clerum.io/managed-by'] = 'someone-else'
+ },
+ ],
+ [
+ 'named after another Deployment',
+ deployment => {
+ deployment.metadata!.name = 'other-host'
+ },
+ ],
+ [
+ 'created for a prior Host incarnation',
+ deployment => {
+ deployment.metadata!.annotations!['clerum.io/host-uid'] = 'prior-host-uid'
+ },
+ ],
+ [
+ 'being deleted',
+ deployment => {
+ deployment.metadata!.deletionTimestamp = new Date('2026-07-02T00:00:00.000Z')
+ },
+ ],
+ [
+ 'without a uid',
+ deployment => {
+ delete deployment.metadata!.uid
+ },
+ ],
+ [
+ 'without a resourceVersion',
+ deployment => {
+ delete deployment.metadata!.resourceVersion
+ },
+ ],
+ [
+ 'without a pod template spec',
+ deployment => {
+ delete deployment.spec!.template!.spec
+ },
+ ],
+ ])('refuses to renew when the Deployment is %s', async (_label, corrupt) => {
+ await expectRefusedAfterReads({
+ deployment: (reconciler, host) => {
+ const deployment = trustedRuntimeDeployment(reconciler, host)
+ corrupt(deployment)
+ return deployment
+ },
+ })
+ })
+
+ it('refuses to renew when the Deployment is absent', async () => {
+ await expectRefusedAfterReads({ deployment: () => null })
+ })
+
+ it.each<[string, (deployment: k8s.V1Deployment) => void]>([
+ [
+ 'scaled to zero replicas',
+ deployment => {
+ deployment.spec!.replicas = 0
+ },
+ ],
+ [
+ 'without a replica count',
+ deployment => {
+ delete deployment.spec!.replicas
+ },
+ ],
+ ])('refuses to renew when the Ready-status Deployment is %s', async (_label, corrupt) => {
+ await expectRefusedAfterReads({
+ deployment: (reconciler, host) => {
+ // readyReplicas stays 1 so only the replica-count guard can refuse.
+ const deployment = trustedRuntimeDeployment(reconciler, host, { readyReplicas: 1 })
+ corrupt(deployment)
+ return deployment
+ },
+ })
+ })
+
+ describe('with an unconsumed wake bootstrap', () => {
+ const APPLIED_REVISION = 'applied-runtime-revision'
+
+ /** A due record whose fresh bootstrap is bound to the given Deployment UID. */
+ const freshBootstrapBoundTo = (boundUid: string) => (host: HostCRD) => {
+ const record = runtimeCredentialRecord(host, dueForRenewal)
+ Object.assign(record.metadata!.annotations!, {
+ 'clerum.io/runtime-token-bootstrap-state': 'fresh',
+ 'clerum.io/runtime-token-bootstrap-deployment-uid': boundUid,
+ 'clerum.io/runtime-token-bootstrap-applied-revision': APPLIED_REVISION,
+ })
+ return record
+ }
+ const appliedDeployment = (reconciler: HostReconciler, host: HostCRD) => {
+ const deployment = trustedRuntimeDeployment(reconciler, host)
+ deployment.spec!.template!.metadata!.annotations = {
+ ...deployment.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': APPLIED_REVISION,
+ }
+ return deployment
+ }
+
+ it('marks a bootstrap bound to the applied Deployment consumed without minting', async () => {
+ const host = makeStatelessHost()
+
+ const bound = await renewHeldGfs({
+ host,
+ record: freshBootstrapBoundTo(`deployment-${host.name}`),
+ deployment: appliedDeployment,
+ })
+
+ // Liveness witness: the record was read and the consumed marker was written.
+ expect(bound.recordReads).toBeGreaterThan(0)
+ expect(bound.recordWrites).toBe(1)
+ expect(bound.provision.revision).toBe(APPLIED_REVISION)
+ expect(bound.mints).toBe(0)
+ })
+
+ it('renews once the bootstrap is bound to a different Deployment UID', async () => {
+ const host = makeStatelessHost()
+ const bound = await renewHeldGfs({
+ host,
+ record: freshBootstrapBoundTo(`deployment-${host.name}`),
+ deployment: appliedDeployment,
+ })
+ expect(bound.mints).toBe(0)
+ expect(bound.recordWrites).toBe(1)
+
+ const replaced = await renewHeldGfs({
+ host,
+ record: freshBootstrapBoundTo('deployment-replaced-uid'),
+ deployment: appliedDeployment,
+ })
+
+ expect(replaced.mints).toBe(1)
+ expect(replaced.recordWrites).toBe(1)
+ })
+ })
+ })
+
it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
@@ -1511,19 +2103,25 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
- await reconciler.reconcile(host)
+ for (let pass = 1; pass <= 3; pass++) {
+ await reconciler.reconcile(host)
- expect(provision).toHaveBeenCalledOnce()
- expect(provision).toHaveBeenCalledWith(host, {
- forceFreshForWake: true,
- targetSuspended: false,
- preserveDeploymentTemplateOnWake: true,
- })
- expect(provision.mock.invocationCallOrder[0]).toBeLessThan(
- appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
- )
- expect(live().spec!.replicas).toBe(1)
- expect(live().spec!.template).toEqual(applied.spec!.template)
+ // One credential mint per pass, made before that pass's replica-only update.
+ expect(provision).toHaveBeenCalledTimes(pass)
+ expect(provision).toHaveBeenLastCalledWith(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ preserveDeploymentTemplateOnWake: true,
+ })
+ const passReplace = appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
+ expect(provision.mock.invocationCallOrder.at(-1)!).toBeLessThan(passReplace)
+ expect(live().spec!.replicas).toBe(1)
+ expect(live().spec!.template).toEqual(applied.spec!.template)
+
+ // Scale the held Deployment back to zero so the next pass recovers it again.
+ live().spec!.replicas = 0
+ live().status = { readyReplicas: 0 }
+ }
})
it('does not roll a held wake bootstrap when the watch recovers before Ready', async () => {
@@ -1655,6 +2253,150 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(second.revision).toBe('applied-runtime-revision')
})
+ describe('wake bootstrap binding to the applied Deployment', () => {
+ const BOOTSTRAP_UID = 'clerum.io/runtime-token-bootstrap-deployment-uid'
+ const BOOTSTRAP_REVISION = 'clerum.io/runtime-token-bootstrap-applied-revision'
+ const APPLIED_REVISION = 'applied-runtime-revision'
+
+ /**
+ * Mints a wake bootstrap for a zero-replica Deployment through the real
+ * producer, then models the wake: the same Deployment at one replica that is
+ * not Ready yet, with the persisted record bound to it.
+ */
+ async function boundWakeBootstrap() {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler()
+ const deployment = trustedRuntimeDeployment(reconciler, host, {
+ replicas: 0,
+ readyReplicas: 0,
+ })
+ deployment.spec!.template!.metadata!.annotations = {
+ ...deployment.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': APPLIED_REVISION,
+ }
+ appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
+ coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
+ const issueRuntimeMaterial = vi.mocked(issueMcpHostRuntimeTokens)
+ issueRuntimeMaterial.mockClear()
+
+ await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ preserveDeploymentTemplateOnWake: true,
+ })
+
+ const created = coreApi.createNamespacedSecret.mock.calls.find(([request]) =>
+ request.body?.metadata?.name?.includes('runtime-tokens')
+ )?.[0].body as k8s.V1Secret
+ expect(created.metadata?.annotations).toMatchObject({
+ 'clerum.io/runtime-token-bootstrap-state': 'fresh',
+ [BOOTSTRAP_UID]: deployment.metadata!.uid,
+ [BOOTSTRAP_REVISION]: APPLIED_REVISION,
+ })
+ const record = withReadableRuntimeRefreshMaterial(created)
+ coreApi.readNamespacedSecret.mockResolvedValue(record)
+ deployment.spec!.replicas = 1
+
+ /** One follow-up provision on the woken Deployment, with the counters it moved. */
+ const provisionOnWokenDeployment = async () => {
+ issueRuntimeMaterial.mockClear()
+ coreApi.replaceNamespacedSecret.mockClear()
+ const recordReadsBefore = coreApi.readNamespacedSecret.mock.calls.length
+ const deploymentReadsBefore = appsApi.readNamespacedDeployment.mock.calls.length
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ targetSuspended: false,
+ preserveDeploymentTemplateOnWake: true,
+ })
+ return {
+ provision,
+ mints: issueRuntimeMaterial.mock.calls.length,
+ recordReads: coreApi.readNamespacedSecret.mock.calls.length - recordReadsBefore,
+ deploymentReads:
+ appsApi.readNamespacedDeployment.mock.calls.length - deploymentReadsBefore,
+ writes: coreApi.replaceNamespacedSecret.mock.calls.map(([request]) => request.body),
+ }
+ }
+ return { deployment, record, provisionOnWokenDeployment }
+ }
+
+ it('reuses a wake bootstrap bound to the applied Deployment UID and revision', async () => {
+ const { provisionOnWokenDeployment } = await boundWakeBootstrap()
+
+ const reused = await provisionOnWokenDeployment()
+
+ // Liveness witness: the record and the Deployment were read and the reuse
+ // path returned the applied revision it binds.
+ expect(reused.recordReads).toBeGreaterThan(0)
+ expect(reused.deploymentReads).toBeGreaterThan(0)
+ expect(reused.provision.revision).toBe(APPLIED_REVISION)
+ expect(reused.mints).toBe(0)
+ expect(reused.writes).toHaveLength(0)
+ })
+
+ it.each<[string, (deployment: k8s.V1Deployment) => void]>([
+ [
+ 'Deployment UID',
+ deployment => {
+ deployment.metadata!.uid = 'deployment-replaced-uid'
+ },
+ ],
+ [
+ 'applied runtime revision',
+ deployment => {
+ deployment.spec!.template!.metadata!.annotations = {
+ ...deployment.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'other-applied-revision',
+ }
+ },
+ ],
+ ])('mints again once the bound %s changes', async (_label, changeDeployment) => {
+ const { deployment, provisionOnWokenDeployment } = await boundWakeBootstrap()
+ const reused = await provisionOnWokenDeployment()
+ expect(reused.recordReads).toBeGreaterThan(0)
+ expect(reused.mints).toBe(0)
+
+ changeDeployment(deployment)
+ const reissued = await provisionOnWokenDeployment()
+
+ expect(reissued.mints).toBe(1)
+ expect(reissued.writes).toHaveLength(1)
+ expect((reissued.writes[0] as k8s.V1Secret).metadata?.annotations).toMatchObject({
+ 'clerum.io/runtime-token-bootstrap-state': 'fresh',
+ })
+ })
+
+ it.each<[string, (deployment: k8s.V1Deployment, record: k8s.V1Secret) => void]>([
+ [
+ 'empty Deployment UID',
+ (deployment, record) => {
+ deployment.metadata!.uid = ''
+ record.metadata!.annotations![BOOTSTRAP_UID] = ''
+ },
+ ],
+ [
+ 'empty applied runtime revision',
+ (deployment, record) => {
+ deployment.spec!.template!.metadata!.annotations = {
+ ...deployment.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': '',
+ }
+ record.metadata!.annotations![BOOTSTRAP_REVISION] = ''
+ },
+ ],
+ ])('does not treat an %s on both sides as a binding', async (_label, empty) => {
+ const { deployment, record, provisionOnWokenDeployment } = await boundWakeBootstrap()
+ const reused = await provisionOnWokenDeployment()
+ expect(reused.recordReads).toBeGreaterThan(0)
+ expect(reused.mints).toBe(0)
+
+ empty(deployment, record)
+ const reissued = await provisionOnWokenDeployment()
+
+ expect(reissued.mints).toBe(1)
+ expect(reissued.writes).toHaveLength(1)
+ })
+ })
+
it('keeps an active held Deployment at zero when credential minting is unavailable', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
@@ -2888,6 +3630,71 @@ describe('HostReconciler stateless lifecycle — suspension durability', () => {
})
})
+describe('HostReconciler readiness poll generation fence', () => {
+ afterEach(() => {
+ vi.useRealTimers()
+ })
+
+ async function pollOnce(options: { supersede: boolean }) {
+ vi.useFakeTimers()
+ const { reconciler } = createReconciler()
+ const internals = reconciler as unknown as {
+ advanceReconcileGeneration(name: string): number
+ setStatus(name: string, status: { deployed: boolean; ready: boolean; message: string }): void
+ pollReadiness(
+ name: string,
+ namespace: string,
+ intervalMs?: number,
+ maxAttempts?: number
+ ): void
+ }
+ const readySpy = vi.spyOn(reconciler as any, 'checkDeploymentReady').mockResolvedValue(true)
+ internals.advanceReconcileGeneration('stateless-host')
+ internals.setStatus('stateless-host', {
+ deployed: true,
+ ready: false,
+ message: 'Deployed, waiting for readiness',
+ })
+ internals.pollReadiness('stateless-host', 'tenant-ns', 10, 2)
+ if (options.supersede) {
+ // A later reconcile pass owns the status now; the older poll must not
+ // overwrite its verdict with a stale 'Running'.
+ internals.advanceReconcileGeneration('stateless-host')
+ internals.setStatus('stateless-host', {
+ deployed: true,
+ ready: false,
+ message: 'Degraded: newer reconcile verdict',
+ })
+ }
+ await vi.advanceTimersByTimeAsync(10)
+ return { reconciler, readySpy }
+ }
+
+ it('reports Running when no newer reconcile superseded the poll', async () => {
+ const { reconciler, readySpy } = await pollOnce({ supersede: false })
+
+ expect(readySpy).toHaveBeenCalledTimes(1)
+ expect(reconciler.getStatus('stateless-host')).toEqual({
+ deployed: true,
+ ready: true,
+ message: 'Running',
+ })
+ })
+
+ it('keeps the newer reconcile verdict when the generation advanced during the poll', async () => {
+ const { reconciler, readySpy } = await pollOnce({ supersede: true })
+
+ // Liveness witness: the poll ran and observed a ready Deployment.
+ expect(readySpy).toHaveBeenCalledTimes(1)
+ expect(readySpy).toHaveBeenCalledWith('stateless-host', 'tenant-ns')
+ expect(reconciler.getStatus('stateless-host')).toEqual({
+ deployed: true,
+ ready: false,
+ message: 'Degraded: newer reconcile verdict',
+ })
+ })
+})
+
describe('HostReconciler stateless lifecycle — probe tuning (Stage 6, W7)', () => {
it('gives the stateless pod an aggressive startup probe preserving the 310s allowance', () => {
const { reconciler } = createReconciler()
diff --git a/host-context-controller/test/statelessLifecycleTracker.test.ts b/host-context-controller/test/statelessLifecycleTracker.test.ts
index b540c2bc0..ec9e1494c 100644
--- a/host-context-controller/test/statelessLifecycleTracker.test.ts
+++ b/host-context-controller/test/statelessLifecycleTracker.test.ts
@@ -210,6 +210,69 @@ describe('StatelessLifecycleTracker — D8 idle rule', () => {
expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
})
+ it('fresh-checks a suspension-blocked Host once per channel authority loss epoch', async () => {
+ const port = makePort()
+ const blocked = { stateless: true, state: 'active', suspensionBlocked: true }
+ port.getEffectiveLifecycle.mockReturnValue(blocked)
+ const tracker = makeTracker({ port, host: makeHost() })
+
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(1)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
+
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(2)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
+
+ port.getEffectiveLifecycle.mockReturnValue({ stateless: true, state: 'active' })
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(3)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(1)
+
+ port.getEffectiveLifecycle.mockReturnValue(blocked)
+ await expect(tracker.handleHeartbeat(payload())).resolves.toEqual({ drain: false })
+ expect(port.getEffectiveLifecycle).toHaveBeenCalledTimes(4)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
+ expect(port.suspendHostFromHeartbeat).not.toHaveBeenCalled()
+ })
+
+ it('retries a failed suspension-blocked fresh check and still reverts cached draining evidence', async () => {
+ const port = makePort()
+ port.getEffectiveLifecycle.mockReturnValue({
+ stateless: true,
+ state: 'active',
+ suspensionBlocked: true,
+ })
+ port.markHostActiveFromHeartbeat.mockRejectedValueOnce(new Error('api down'))
+ let cachedHost = makeHost()
+ const tracker = new StatelessLifecycleTracker({
+ idleMinutes: 30,
+ idleFloorMinutes: 15,
+ drainGraceMs: 60_000,
+ maxUptimeHours: 72,
+ reconciler: port as unknown as StatelessLifecycleReconcilerPort,
+ getHost: () => cachedHost,
+ now: () => NOW,
+ })
+ const errors = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+ try {
+ await tracker.handleHeartbeat(payload())
+ await tracker.handleHeartbeat(payload())
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
+ expect(errors).toHaveBeenCalledOnce()
+
+ await tracker.handleHeartbeat(payload())
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(2)
+
+ cachedHost = makeHost({ lifecycle: { state: 'draining', wakeHandledGeneration: 0 } })
+ await tracker.handleHeartbeat(payload())
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenCalledTimes(3)
+ expect(port.markHostActiveFromHeartbeat).toHaveBeenLastCalledWith(cachedHost)
+ } finally {
+ errors.mockRestore()
+ }
+ })
+
it('answers drain:false for an unknown host', async () => {
const port = makePort()
const tracker = new StatelessLifecycleTracker({