Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): hold stateless runtimes with verified scopes

Alfredo Lopez7655ab3d6dee
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 623c9384c..ca132875a 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.360",
+ "version": "0.1.361",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.360",
+ "version": "0.1.361",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 69bebc426..f772b7b60 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.360",
+ "version": "0.1.361",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
index 7f6f24ed4..4865643b9 100644
--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
@@ -172,6 +172,8 @@ function runtimeSecretAnnotations(host = makeHost(), refreshBefore = '2999-01-01
'clerum.io/gfs-token-host-uid': host.uid!,
'clerum.io/runtime-token-refresh-before': refreshBefore,
'clerum.io/gfs-token-refresh-before': '2999-01-01T00:00:00.000Z',
+ 'clerum.io/runtime-token-has-channel-ingress': 'false',
+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',
}
}
@@ -233,6 +235,7 @@ function runtimeSecretReconciler(opts: {
const reconciler = new HostReconciler(makeStubKc(), {
coreApi: coreApi as unknown as k8s.CoreV1Api,
appsApi: appsApi as unknown as k8s.AppsV1Api,
+ isCommunicationChannelCacheSynced: () => true,
})
return { reconciler, coreApi, appsApi }
}
@@ -1256,6 +1259,7 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
const stubKc = makeStubKc()
const reconciler = new HostReconciler(stubKc, {
resolveContextMounts,
+ isCommunicationChannelCacheSynced: () => true,
// Stub APIs to capture the Deployment body and ack everything.
coreApi: {
readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(),
diff --git a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
index cde5f5035..37493ee32 100644
--- a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
+++ b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts
@@ -149,6 +149,7 @@ function createReconciler() {
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
+ isCommunicationChannelCacheSynced: () => true,
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi }
}
diff --git a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts b/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts
index f9a5de272..9811f0da9 100644
--- a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts
+++ b/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts
@@ -168,6 +168,7 @@ function createReconciler() {
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
+ isCommunicationChannelCacheSynced: () => true,
})
return { reconciler, appsApi, coreApi, networkingApi, rbacApi }
}
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 87f5a4c79..ecb01b8c7 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -392,6 +392,8 @@ const RUNTIME_TOKEN_AUDIENCE_ANNOTATION = 'clerum.io/runtime-token-audience'
const RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION = 'clerum.io/runtime-token-schema-version'
const RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION = 'clerum.io/runtime-token-bootstrap-state'
const RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION = 'clerum.io/runtime-token-rollout-required'
+const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'
+const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'
const RUNTIME_TOKEN_ISSUER = 'control-api'
const RUNTIME_TOKEN_AUDIENCE = 'host-context-controller,workflow-approvals'
// v3 extends the existing first-party access/refresh token material with the
@@ -498,6 +500,7 @@ type HostSecretValidationResult =
type BootstrapOptions = {
forceFreshForWake?: boolean
targetSuspended?: boolean
+ refreshGfsOnly?: boolean
}
type RuntimeTokenProvision = {
@@ -988,7 +991,7 @@ export class HostReconciler {
}
}
- private isHccOwnedHostResource(
+ private static isHccOwnedHostResource(
resource: { metadata?: { labels?: Record<string, string> } },
hostName: string
): boolean {
@@ -1013,7 +1016,7 @@ export class HostReconciler {
throw error
}
- if (!this.isHccOwnedHostResource(resource, hostName)) {
+ if (!HostReconciler.isHccOwnedHostResource(resource, hostName)) {
log.warn('Skipping Host resource delete - not HCC-owned', { kind, name, host: hostName })
return
}
@@ -1558,6 +1561,51 @@ export class HostReconciler {
return (deployment?.status?.readyReplicas ?? 0) > 0
}
+ private static credentialRecordBelongsToHost(
+ record: { metadata?: { labels?: Record<string, string> } },
+ host: HostCRD
+ ): boolean {
+ return HostReconciler.isHccOwnedHostResource(record, host.name)
+ }
+
+ private static runtimeRecordIsComplete(
+ record: Pick<k8s.V1Secret, 'data' | 'stringData'>
+ ): boolean {
+ return HostReconciler.runtimeTokenSecretData(record) !== null
+ }
+
+ private static retainedBoolean(record: k8s.V1Secret, key: string): boolean | null {
+ const value = record.metadata?.annotations?.[key]
+ return value === 'true' ? true : value === 'false' ? false : null
+ }
+
+ private static deploymentBelongsToHost(
+ deployment: k8s.V1Deployment | null,
+ host: HostCRD
+ ): boolean {
+ return (
+ !!deployment &&
+ HostReconciler.credentialRecordBelongsToHost(deployment, host) &&
+ deployment.metadata?.name === host.name &&
+ deployment.metadata?.namespace === host.namespace &&
+ !!deployment.metadata?.uid &&
+ !!deployment.metadata?.resourceVersion &&
+ !deployment.metadata?.deletionTimestamp &&
+ !!deployment.spec?.template?.spec &&
+ !!host.uid &&
+ deployment.metadata?.annotations?.[HOST_UID_ANNOTATION] === host.uid
+ )
+ }
+
+ private static credentialRenewalWindowReached(
+ record: k8s.V1Secret,
+ nowMs: number,
+ annotation: string
+ ): boolean {
+ const refreshBefore = Date.parse(record.metadata?.annotations?.[annotation] ?? '')
+ return !Number.isFinite(refreshBefore) || nowMs >= refreshBefore
+ }
+
/**
* True when this reconcile will (re)start a pod that reads the
* mcp-host-runtime-token Secret fresh at boot -- a missing Deployment, a
@@ -1759,7 +1807,7 @@ export class HostReconciler {
private async ensureMcpHostRuntimeTokenSecret(
host: HostCRD,
options: BootstrapOptions = {}
- ): Promise<RuntimeTokenProvision> {
+ ): Promise<RuntimeTokenProvision | null> {
let lastErr: unknown = null
for (let attempt = 1; attempt <= 3; attempt++) {
try {
@@ -1780,22 +1828,141 @@ export class HostReconciler {
const existingRevision = existing
? HostReconciler.runtimeTokenSecretRevisionFromSecret(existing)
: null
+ const nowMs = Date.now()
+ if (options.refreshGfsOnly) {
+ const trustedDeployment =
+ HostReconciler.deploymentBelongsToHost(deployment, host) &&
+ (deployment?.spec?.replicas ?? 0) > 0 &&
+ HostReconciler.deploymentReady(deployment)
+ if (
+ !existing ||
+ !HostReconciler.credentialRecordBelongsToHost(existing, host) ||
+ !host.uid ||
+ existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid ||
+ !HostReconciler.runtimeRecordIsComplete(existing) ||
+ !trustedDeployment
+ ) {
+ log.warn(
+ 'deferring held-runtime credential renewal because runtime identity is not verified',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ }
+ )
+ return null
+ }
+ if (
+ !HostReconciler.credentialRenewalWindowReached(
+ existing,
+ nowMs,
+ GFS_TOKEN_REFRESH_BEFORE_ANNOTATION
+ )
+ ) {
+ this.recordGfsLifecycleEvidence(host, 'reused')
+ return {
+ revision:
+ HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
+ scopeHash:
+ existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
+ }
+ }
+
+ const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
+ const data = { ...(HostReconciler.runtimeTokenSecretData(existing) ?? {}) }
+ data[MCP_HOST_GFS_TOKEN_SECRET_KEY] = Buffer.from(gfs.token).toString('base64')
+ const gfsExpiresAtMs = nowMs + Math.max(0, gfs.expiresInSeconds) * 1000
+ const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(
+ Math.max(0, gfs.expiresInSeconds)
+ )
+ const nextRevision = HostReconciler.runtimeTokenSecretRevision(data)
+ await this.coreApi.replaceNamespacedSecret({
+ name,
+ namespace: host.namespace,
+ body: {
+ ...existing,
+ data,
+ metadata: {
+ ...existing.metadata,
+ annotations: {
+ ...(existing.metadata?.annotations ?? {}),
+ [GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
+ makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
+ [GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
+ [GFS_TOKEN_EXPIRES_AT_ANNOTATION]: new Date(gfsExpiresAtMs).toISOString(),
+ [GFS_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(
+ gfsExpiresAtMs - gfsRefreshBeforeSec * 1000
+ ).toISOString(),
+ [RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: nextRevision,
+ },
+ },
+ },
+ })
+ this.recordGfsLifecycleEvidence(host, 'rotated')
+ return {
+ revision:
+ HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',
+ scopeHash: existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
+ }
+ }
const bootstrapIsFresh =
existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
- const nowMs = Date.now()
- const hasChannelIngress = this.hasChannelIngress(host)
+ const retainedAnnotations = existing?.metadata?.annotations ?? {}
+ const cacheSynced = this.ccCacheSyncedFn()
+ const retainedScopeIsTrusted =
+ !!existing &&
+ HostReconciler.credentialRecordBelongsToHost(existing, host) &&
+ !!host.uid &&
+ retainedAnnotations[GFS_TOKEN_HOST_UID_ANNOTATION] === host.uid
+ const retainedChannelIngress =
+ retainedScopeIsTrusted && existing
+ ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION)
+ : null
+ const retainedFrontsOAuth =
+ retainedScopeIsTrusted && existing
+ ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION)
+ : null
+ const hasChannelIngress = cacheSynced
+ ? this.hasChannelIngress(host)
+ : retainedChannelIngress === true
+ if (!cacheSynced && retainedChannelIngress === null) {
+ log.warn(
+ 'skipping runtime token mint during channel cache loss without retained scope observation',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ observation: 'hasChannelIngress',
+ }
+ )
+ return null
+ }
// Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and
// codex projection into the mint-scope derive, the refresh decision, the
// scope hash and the stored annotation, so the minted token and the drift
// hash never diverge.
- const frontsOAuthServer = await this.frontsOAuthServer(host)
+ let observedFrontsOAuth: boolean
+ if (cacheSynced) {
+ observedFrontsOAuth = await this.frontsOAuthServer(host)
+ } else {
+ if (retainedFrontsOAuth === null) {
+ log.warn(
+ 'skipping runtime token mint during channel cache loss without retained OAuth observation',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ observation: 'frontsOAuthServer',
+ }
+ )
+ return null
+ }
+ observedFrontsOAuth = retainedFrontsOAuth
+ }
const projection = this.projectCodexForHost(host)
const grokProjection = this.projectGrokForHost(host)
const scopeHash = HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
- frontsOAuthServer,
+ observedFrontsOAuth,
projection,
grokProjection
)
@@ -1810,7 +1977,7 @@ export class HostReconciler {
existing,
nowMs,
hasChannelIngress,
- frontsOAuthServer,
+ observedFrontsOAuth,
projection,
grokProjection
)
@@ -1870,6 +2037,19 @@ export class HostReconciler {
deploymentRevision !== '' &&
deploymentRevision !== existingRevision)
const annotationUpdates: Record<string, string> = {}
+ if (cacheSynced) {
+ const currentIngress =
+ existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]
+ if (currentIngress !== String(hasChannelIngress)) {
+ annotationUpdates[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION] =
+ String(hasChannelIngress)
+ }
+ const currentOAuth =
+ existing.metadata?.annotations?.[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]
+ if (currentOAuth !== String(observedFrontsOAuth)) {
+ annotationUpdates[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION] = String(observedFrontsOAuth)
+ }
+ }
if (
host.generation !== undefined &&
existing.metadata?.annotations?.[GFS_TOKEN_HOST_GENERATION_ANNOTATION] !==
@@ -1950,7 +2130,7 @@ export class HostReconciler {
const tokens = await issueMcpHostRuntimeTokens(
host.name,
host.uid ?? '',
- this.resolveEffectiveControlScopesForHost(host, frontsOAuthServer, hasChannelIngress)
+ this.resolveEffectiveControlScopesForHost(host, observedFrontsOAuth, hasChannelIngress)
)
const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })
const body = buildMcpHostRuntimeTokenSecret(
@@ -1978,13 +2158,15 @@ export class HostReconciler {
nowMs,
gfs.expiresInSeconds,
hasChannelIngress,
- frontsOAuthServer,
+ observedFrontsOAuth,
existing?.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION],
projection,
grokProjection
),
[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,
[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',
+ [RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),
+ [RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]: String(observedFrontsOAuth),
},
}
@@ -2055,7 +2237,7 @@ export class HostReconciler {
private async provisionRuntimeTokenRevision(
host: HostCRD,
options: BootstrapOptions
- ): Promise<RuntimeTokenProvision> {
+ ): Promise<RuntimeTokenProvision | null> {
try {
return await this.ensureMcpHostRuntimeTokenSecret(host, options)
} catch (err) {
@@ -3653,7 +3835,7 @@ export class HostReconciler {
if (!observedDeployment) return null
const existing = observedDeployment
if (
- !this.isHccOwnedHostResource(existing, host.name) ||
+ !HostReconciler.isHccOwnedHostResource(existing, host.name) ||
existing.metadata?.name !== host.name ||
existing.metadata?.namespace !== host.namespace ||
!existing.metadata?.uid ||
@@ -4800,17 +4982,53 @@ export class HostReconciler {
return false
}
}
- const holdAppliedRuntime = async (allowScaleUp: boolean): Promise<void> => {
- if (this.ccCacheSyncedFn()) {
- await this.provisionRuntimeTokenRevision(host, {
- forceFreshForWake: allowScaleUp,
+ const holdAppliedRuntime = async (wakeRequested: boolean): Promise<void> => {
+ const liveDeployment = await this.readHostDeploymentOrNull(host)
+ const deploymentIdentityIsTrusted =
+ !liveDeployment || HostReconciler.deploymentBelongsToHost(liveDeployment, host)
+ const runtimeIsReady =
+ !!liveDeployment &&
+ HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&
+ HostReconciler.deploymentReady(liveDeployment)
+ const scaleUpForRecovery =
+ lifecycle.effective.state === 'active' &&
+ (!liveDeployment || (liveDeployment.spec?.replicas ?? 1) === 0)
+ const allowScaleUp = deploymentIdentityIsTrusted && (wakeRequested || scaleUpForRecovery)
+ let provisioningRequired = false
+ if (allowScaleUp) {
+ // A pod starting from zero will consume bootstrap credentials. Mint and
+ // persist them before the replica-only update; a failed mint aborts here.
+ provisioningRequired = true
+ } else if (lifecycle.effective.state === 'active' && runtimeIsReady) {
+ const provision = await this.provisionRuntimeTokenRevision(host, {
+ targetSuspended: false,
+ refreshGfsOnly: true,
+ })
+ if (!provision) {
+ this.setStatus(host.name, {
+ deployed: true,
+ ready: false,
+ message: 'Held runtime credential renewal is not safely available',
+ })
+ return
+ }
+ revalidateHostMutationBoundary()
+ }
+ if (provisioningRequired) {
+ const provision = await this.provisionRuntimeTokenRevision(host, {
+ forceFreshForWake: true,
targetSuspended: lifecycle.effective.state === 'suspended',
})
+ if (!provision) {
+ this.setStatus(host.name, {
+ deployed: !!liveDeployment,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ return
+ }
revalidateHostMutationBoundary()
}
- // The channel inventory is incomplete while its cache is unsynced. Defer
- // scope-sensitive token minting until the next authoritative reconcile so
- // a missing channel count cannot narrow the running Host's token scopes.
const applied = await this.ensureDeployment(
host,
mounts,
@@ -4961,6 +5179,23 @@ export class HostReconciler {
forceFreshForWake,
targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
+ if (!runtimeTokenProvision) {
+ const liveDeployment = await this.readHostDeploymentOrNull(host)
+ this.setStatus(host.name, {
+ deployed: !!liveDeployment,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ return
+ }
+ const requireRuntimeTokenProvision = (): RuntimeTokenProvision => {
+ if (!runtimeTokenProvision) {
+ const error = new Error('Waiting for authoritative scope observation')
+ error.name = 'RuntimeScopeObservationUnavailableError'
+ throw error
+ }
+ return runtimeTokenProvision
+ }
revalidateHostMutationBoundary()
// replaceWithConflictRetry may wait and re-read after a 409, so each body is
@@ -4984,19 +5219,24 @@ export class HostReconciler {
for (let attempt = 1; attempt <= 3; attempt++) {
revalidateHostMutationBoundary()
const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))
- if (runtimeTokenProvision.scopeHash === currentScopeHash) return
+ if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) return
runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake: false,
targetSuspended:
lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',
})
+ if (!runtimeTokenProvision) {
+ const error = new Error('Waiting for authoritative scope observation')
+ error.name = 'RuntimeScopeObservationUnavailableError'
+ throw error
+ }
revalidateHostMutationBoundary()
const postProvisionScopeHash = this.runtimeScopeHashFor(
host,
await this.frontsOAuthServer(host)
)
- if (runtimeTokenProvision.scopeHash === postProvisionScopeHash) return
+ if (requireRuntimeTokenProvision().scopeHash === postProvisionScopeHash) return
log.warn('CommunicationChannel scope contract changed during token provisioning', {
host: host.name,
@@ -5016,7 +5256,7 @@ export class HostReconciler {
if (beforeScope.suspensionBlocked) {
return {
lifecycle: beforeScope,
- runtimeTokenRevision: runtimeTokenProvision.revision,
+ runtimeTokenRevision: requireRuntimeTokenProvision().revision,
grokExecutionEnabled: this.hostDerivesGrokExecution(host),
}
}
@@ -5031,11 +5271,11 @@ export class HostReconciler {
) {
continue
}
- if (runtimeTokenProvision.scopeHash === currentScopeHash) {
+ if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) {
revalidateHostMutationBoundary()
return {
lifecycle: effective,
- runtimeTokenRevision: runtimeTokenProvision.revision,
+ runtimeTokenRevision: requireRuntimeTokenProvision().revision,
grokExecutionEnabled: this.hostDerivesGrokExecution(host),
}
}
@@ -5051,14 +5291,28 @@ export class HostReconciler {
}
revalidateHostMutationBoundary()
- const deploymentApplied = await this.ensureDeployment(
- host,
- mounts,
- runtimeTokenProvision.revision,
- lifecycle.effective,
- resolveDeploymentState,
- revalidateHostMutationBoundary
- )
+ let deploymentApplied: boolean
+ try {
+ deploymentApplied = await this.ensureDeployment(
+ host,
+ mounts,
+ runtimeTokenProvision.revision,
+ lifecycle.effective,
+ resolveDeploymentState,
+ revalidateHostMutationBoundary
+ )
+ } catch (error) {
+ if (error instanceof Error && error.name === 'RuntimeScopeObservationUnavailableError') {
+ const liveDeployment = await this.readHostDeploymentOrNull(host)
+ this.setStatus(host.name, {
+ deployed: !!liveDeployment,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ return
+ }
+ throw error
+ }
revalidateHostMutationBoundary()
if (!deploymentApplied) {
diff --git a/host-context-controller/src/statelessLifecycle.types.ts b/host-context-controller/src/statelessLifecycle.types.ts
index 6f7b098fe..ebaa1410f 100644
--- a/host-context-controller/src/statelessLifecycle.types.ts
+++ b/host-context-controller/src/statelessLifecycle.types.ts
@@ -17,7 +17,7 @@ export interface EffectiveHostLifecycle {
state: HostLifecycleState
/** Unknown channel authority: preserve the entire applied Deployment template. */
suspensionBlocked?: boolean
- /** A fresh-read-verified wake may scale the held Deployment without a template change. */
+ /** An active recovery or fresh-read-verified wake may scale the held Deployment without a template change. */
allowScaleUpDuringHold?: boolean
}
diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.ts
index 4647ce538..f556dced4 100644
--- a/host-context-controller/src/statelessLifecycleExecutor.ts
+++ b/host-context-controller/src/statelessLifecycleExecutor.ts
@@ -231,7 +231,11 @@ export class StatelessLifecycleExecutor {
return { stateless: false, state: 'active' }
}
if (!this.isCommunicationChannelCacheSynced()) {
- return { stateless: true, state: 'active', suspensionBlocked: true }
+ return {
+ stateless: true,
+ state: host.status?.lifecycle?.state ?? 'active',
+ suspensionBlocked: true,
+ }
}
return { stateless: true, state: host.status?.lifecycle?.state ?? 'active' }
}
@@ -1790,8 +1794,8 @@ export class StatelessLifecycleExecutor {
assessment: HostLifecycleAssessment
): Promise<HostLifecycleAssessment | null> {
if (assessment.effective.suspensionBlocked) {
- // Cache authority loss intentionally forces active even when the fresh
- // durable state is suspended; there is no scale-down to validate.
+ // Cache authority loss blocks lifecycle-driven scale-down; preserve the
+ // durable state for accurate reporting without mutating the Deployment.
return assessment
}
if (!assessment.effective.stateless) {
diff --git a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
index b0e5605c3..daf609c93 100644
--- a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
+++ b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
@@ -205,48 +205,28 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synce
})
})
-// ── B2: unsynced cache + existing Deployment has replicas=1 → must preserve ──
-describe('reconcile() — B2 integration: replica preservation when CC cache is unsynced (409 path)', () => {
- it('preserves replicas=1 on replaceNamespacedDeployment body when cache is unsynced and existing Deployment is live', async () => {
- // Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced).
- // The existing live Deployment has replicas=1 (channel-reader is running).
- // reconcile() MUST NOT scale it to 0 — that would drop inbound messages.
+// ── B2: unsynced scope with no retained observation must defer Deployment ──
+describe('reconcile() — B2 integration: Deployment is deferred without an authoritative scope observation', () => {
+ it('does not mutate Deployment when the CC cache is unsynced and scope was not retained', async () => {
+ // Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced),
+ // and the runtime Secret has no trusted retained scope observation. The
+ // reconciler must wait before changing the runtime Deployment.
const { reconciler, appsApi } = createReconciler()
- // Force the 409 → read-existing → replace path ONLY for the channels namespace
- // channel-reader Deployment. The mcp-host Deployment (mcp-host ns) must succeed
- // so that reconcile() progresses to the channel-reader step.
+ // A Deployment create would expose the runtime to credentials minted from
+ // an incomplete channel inventory; make any attempted create fail loudly.
appsApi.createNamespacedDeployment.mockImplementation(
({ body }: { namespace: string; body: k8s.V1Deployment }) => {
const dep = body as k8s.V1Deployment
- if (dep.metadata?.namespace === 'channels') {
- return Promise.reject({ code: 409 })
- }
- // mcp-host Deployment (mcp-host ns) and everything else succeeds normally
- return Promise.resolve({})
+ return Promise.reject(
+ new Error(`Unexpected Deployment create in ${dep.metadata?.namespace}`)
+ )
}
)
- // readNamespacedDeployment is called for both the host (name=alpha-host, ns=mcp-host)
- // and the channel-reader (name=channel-reader-alpha-host, ns=channels).
- // Return a "live replicas=1" response for the channel-reader read; for the host
- // deployment read, return HCC-owned labels so reconcile() continues normally.
+ // Return an existing live, HCC-owned runtime Deployment. It must remain
+ // untouched while the reconciler waits for authoritative channel scope.
appsApi.readNamespacedDeployment.mockImplementation(
({ name, namespace }: { name: string; namespace: string }) => {
- if (name === 'channel-reader-alpha-host') {
- return Promise.resolve({
- metadata: {
- name: 'channel-reader-alpha-host',
- namespace: 'channels',
- resourceVersion: '77',
- labels: {
- 'clerum.io/host': 'alpha-host',
- 'clerum.io/managed-by': 'host-context-controller',
- },
- },
- spec: { replicas: 1 },
- })
- }
- // Default: HCC-owned host Deployment (for readiness check)
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
@@ -270,15 +250,13 @@ describe('reconcile() — B2 integration: replica preservation when CC cache is
await reconciler.reconcile(makeHost())
- // The replace call body must carry replicas=1, not 0
- expect(
- appsApi.replaceNamespacedDeployment.mock.calls.filter(
- ([request]) => request.namespace === 'channels'
- )
- ).toHaveLength(1)
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
- const replaceBody = getChannelReaderReplaceBody(appsApi)
- expect(replaceBody.spec?.replicas).toBe(1)
+ expect(reconciler.getStatus('alpha-host')).toMatchObject({
+ deployed: true,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
})
})
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 4661a4c01..76e5a8980 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -5,11 +5,18 @@ import { mintHostGfsToken } from '../src/gfsHostBinding'
import {
type EffectiveHostLifecycle,
HostReconciler,
+ OAUTH_USER_TOKEN_SCOPE,
type ResolvedSfsMount,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
import { HostContextLogger } from '../src/logger'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
+import {
+ MCP_HOST_GFS_TOKEN_SECRET_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
+} from '../src/secretFactory'
import { HostCRD, HostCrdStatus } from '../src/types'
import {
type MockAppsApi,
@@ -195,6 +202,98 @@ function runtimeTokenProvision(host: HostCRD, hasChannelIngress = false) {
}
}
+function runtimeCredentialRecord(
+ host: HostCRD,
+ options: {
+ gfsRefreshBefore?: string
+ frontsOAuthServer?: string
+ hasChannelIngress?: string
+ hostUid?: string
+ managedByHost?: boolean
+ } = {}
+) {
+ const data = {
+ [MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]: Buffer.from('access-value').toString('base64'),
+ [MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]: Buffer.from('refresh-value').toString('base64'),
+ [MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]: Buffer.from('control-value').toString('base64'),
+ [MCP_HOST_GFS_TOKEN_SECRET_KEY]: Buffer.from('gfs-value').toString('base64'),
+ }
+ const revision = (
+ HostReconciler as unknown as {
+ runtimeTokenSecretRevision(data: Record<string, string>): string
+ }
+ ).runtimeTokenSecretRevision(data)
+ return {
+ metadata: {
+ name: `host-${host.name}-mcp-host-runtime-tokens`,
+ namespace: host.namespace,
+ resourceVersion: '88',
+ labels:
+ options.managedByHost === false
+ ? { 'clerum.io/managed-by': 'someone-else' }
+ : {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/host': host.name,
+ },
+ annotations: {
+ 'clerum.io/gfs-token-host-uid': options.hostUid ?? host.uid ?? '',
+ 'clerum.io/runtime-token-secret-revision': revision,
+ 'clerum.io/runtime-token-has-channel-ingress': options.hasChannelIngress ?? 'false',
+ 'clerum.io/runtime-token-fronts-oauth-server': options.frontsOAuthServer ?? 'false',
+ 'clerum.io/gfs-token-refresh-before':
+ options.gfsRefreshBefore ?? '2999-01-01T00:00:00.000Z',
+ },
+ },
+ data,
+ } as unknown as k8s.V1Secret
+}
+
+function trustedRuntimeDeployment(
+ reconciler: HostReconciler,
+ host: HostCRD,
+ options: { readyReplicas?: number; replicas?: number } = {}
+) {
+ const deployment = reconciler.buildDeployment(host)
+ deployment.metadata = {
+ ...deployment.metadata,
+ uid: `deployment-${host.name}`,
+ resourceVersion: '91',
+ labels: {
+ ...(deployment.metadata?.labels ?? {}),
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/host': host.name,
+ },
+ annotations: {
+ ...(deployment.metadata?.annotations ?? {}),
+ 'clerum.io/host-uid': host.uid ?? '',
+ },
+ }
+ deployment.spec = { ...deployment.spec, replicas: options.replicas ?? 1 }
+ deployment.status = { readyReplicas: options.readyReplicas ?? 1 }
+ return deployment
+}
+
+function markPersistedRuntimeTrusted(
+ deployment: k8s.V1Deployment,
+ host: HostCRD,
+ readyReplicas = 1
+): k8s.V1Deployment {
+ deployment.metadata = {
+ ...deployment.metadata,
+ labels: {
+ ...(deployment.metadata?.labels ?? {}),
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/host': host.name,
+ },
+ annotations: {
+ ...(deployment.metadata?.annotations ?? {}),
+ 'clerum.io/host-uid': host.uid ?? '',
+ },
+ }
+ deployment.status = { readyReplicas }
+ return deployment
+}
+
/** Model persisted API state so a loss-of-authority test cannot invent a template. */
function persistHostDeployment(appsApi: MockAppsApi, host: HostCRD, initial: k8s.V1Deployment) {
let live = structuredClone(initial)
@@ -313,12 +412,12 @@ describe('HostReconciler stateless lifecycle — buildDeployment replicas', () =
expect(dep.spec?.replicas).toBe(1)
})
- it('forces stateless+suspended to replicas=1 while the channel cache is unsynced', () => {
+ it('preserves stateless+suspended replicas while the channel cache is unsynced', () => {
const { reconciler } = createReconciler({
isCommunicationChannelCacheSynced: () => false,
})
const dep = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))
- expect(dep.spec?.replicas).toBe(1)
+ expect(dep.spec?.replicas).toBe(0)
expect(dep.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
})
@@ -811,6 +910,10 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => false,
})
+ const read = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ request.name === host.name ? Promise.reject({ code: 404 }) : read(request)
+ )
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = createReconciler().reconciler.buildDeployment(host)
applied.metadata = { ...applied.metadata, uid: 'deployment-uid', resourceVersion: '74' }
@@ -971,7 +1074,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
- it('preserves runtime credentials and the existing template during cache loss', async () => {
+ it('refreshes only the GFS token for a running held Host during cache loss', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
@@ -981,12 +1084,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const applied = reconciler.buildDeployment(host)
applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
+ markPersistedRuntimeTrusted(applied, host)
const live = persistHostDeployment(appsApi, host, applied)
const provision = vi
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
- expect(provision).not.toHaveBeenCalled()
+ expect(provision).toHaveBeenCalledOnce()
+ expect(provision).toHaveBeenCalledWith(host, {
+ refreshGfsOnly: true,
+ targetSuspended: false,
+ })
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
@@ -1032,7 +1140,14 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
.spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
.mockResolvedValue(runtimeTokenProvision(host))
await reconciler.reconcile(host)
- expect(provision).not.toHaveBeenCalled()
+ expect(provision).toHaveBeenCalledOnce()
+ expect(provision).toHaveBeenCalledWith(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ })
+ expect(provision.mock.invocationCallOrder[0]).toBeLessThan(
+ appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
+ )
expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()
expect(live().spec!.replicas).toBe(1)
expect(live().spec!.template).toEqual(applied.spec!.template)
@@ -1088,11 +1203,26 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
it('preserves an independently confirmed desktop rejection while the channel cache is unknown', async () => {
const host = makeStatelessHost({ spec: { desktop: { browser: true } } })
- const { reconciler, appsApi, customApi } = createReconciler({
+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({
countCommunicationChannels: () => 1,
isCommunicationChannelCacheSynced: () => false,
})
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ coreApi.readNamespacedSecret.mockResolvedValue({
+ metadata: {
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/host': host.name,
+ },
+ annotations: {
+ 'clerum.io/gfs-token-host-uid': host.uid,
+ 'clerum.io/runtime-token-has-channel-ingress': 'true',
+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',
+ },
+ },
+ data: {},
+ } as any)
await reconciler.reconcile(host)
const condition = rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!)
expect(condition.reason).toBe('DesktopEnabled')
@@ -1104,6 +1234,270 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
).toBe(false)
})
+ it('defers a confirmed desktop rejection when no trusted scope observation exists', async () => {
+ const host = makeStatelessHost({ spec: { desktop: { browser: true } } })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ countCommunicationChannels: () => 1,
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ request.name === host.name ? Promise.reject({ code: 404 }) : readDeployment(request)
+ )
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+
+ await reconciler.reconcile(host)
+
+ expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
+ 'DesktopEnabled'
+ )
+ expect(
+ [
+ ...appsApi.createNamespacedDeployment.mock.calls,
+ ...appsApi.replaceNamespacedDeployment.mock.calls,
+ ].filter(
+ ([request]) => (request as { body?: k8s.V1Deployment }).body?.metadata?.name === host.name
+ )
+ ).toHaveLength(0)
+ expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })
+ })
+
+ it('does not let a retained OAuth observation override a failed live probe', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => true,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ runtimeCredentialRecord(host, { frontsOAuthServer: 'true' })
+ )
+ reconciler.setHostFrontsOAuthServer(async () => {
+ throw new Error('oauth lookup unavailable')
+ })
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
+
+ expect(provision).not.toBeNull()
+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).not.toContain(
+ OAUTH_USER_TOKEN_SCOPE
+ )
+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
+ expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(
+ 'false'
+ )
+ })
+
+ it('uses only retained scope observations while channel authority is unavailable', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, coreApi } = createReconciler({
+ countCommunicationChannels: () => 0,
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ runtimeCredentialRecord(host, {
+ frontsOAuthServer: 'true',
+ hasChannelIngress: 'true',
+ })
+ )
+ const oauthProbe = vi
+ .spyOn(reconciler as any, 'frontsOAuthServer')
+ .mockImplementation(async () => {
+ throw new Error('live lookup must not run')
+ })
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
+
+ expect(provision).not.toBeNull()
+ expect(oauthProbe).not.toHaveBeenCalled()
+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).toContain(
+ OAUTH_USER_TOKEN_SCOPE
+ )
+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
+ expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(
+ 'true'
+ )
+ })
+
+ it('does not renew held-runtime GFS before its refresh window', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(runtimeCredentialRecord(host))
+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ expect(provision).not.toBeNull()
+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
+ })
+
+ it('renews only held-runtime GFS at the refresh boundary', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ const record = runtimeCredentialRecord(host, {
+ gfsRefreshBefore: '2000-01-01T00:00:00.000Z',
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(record)
+ const deployment = trustedRuntimeDeployment(reconciler, host)
+ deployment.spec!.template!.metadata!.annotations = {
+ ...deployment.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',
+ }
+ appsApi.readNamespacedDeployment.mockResolvedValue(deployment)
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()
+ expect(provision.revision).toBe('applied-runtime-revision')
+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]).toBe(
+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]
+ )
+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]).toBe(
+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]
+ )
+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]).toBe(
+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]
+ )
+ expect(write.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(
+ record.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]
+ )
+ })
+
+ it('never falls back to full issuance from a closed held-runtime GFS renewal', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ expect(provision).toBeNull()
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
+ })
+
+ it('does not treat a requested but unready held runtime as ready for GFS renewal', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ runtimeCredentialRecord(host, { gfsRefreshBefore: '2000-01-01T00:00:00.000Z' })
+ )
+ appsApi.readNamespacedDeployment.mockResolvedValue(
+ trustedRuntimeDeployment(reconciler, host, { readyReplicas: 0 })
+ )
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ expect(provision).toBeNull()
+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
+ })
+
+ it('does not reassign a prior Host UID during held-runtime GFS renewal', async () => {
+ const host = makeStatelessHost()
+ const { reconciler, appsApi, coreApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ coreApi.readNamespacedSecret.mockResolvedValue(
+ runtimeCredentialRecord(host, { hostUid: 'prior-host-uid' })
+ )
+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))
+ vi.mocked(mintHostGfsToken).mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {
+ refreshGfsOnly: true,
+ })
+
+ expect(provision).toBeNull()
+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()
+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
+ })
+
+ it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }
+ const live = persistHostDeployment(appsApi, host, applied)
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(runtimeTokenProvision(host))
+
+ await reconciler.reconcile(host)
+
+ expect(provision).toHaveBeenCalledOnce()
+ expect(provision).toHaveBeenCalledWith(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ })
+ expect(provision.mock.invocationCallOrder[0]).toBeLessThan(
+ appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!
+ )
+ expect(live().spec!.replicas).toBe(1)
+ expect(live().spec!.template).toEqual(applied.spec!.template)
+ })
+
+ it('keeps an active held Deployment at zero when credential minting is unavailable', async () => {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const { reconciler, appsApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ const applied = reconciler.buildDeployment(host)
+ applied.spec!.replicas = 0
+ markPersistedRuntimeTrusted(applied, host, 0)
+ const live = persistHostDeployment(appsApi, host, applied)
+ appsApi.replaceNamespacedDeployment.mockClear()
+ const provision = vi
+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')
+ .mockResolvedValue(null)
+
+ await reconciler.reconcile(host)
+
+ expect(provision).toHaveBeenCalledOnce()
+ expect(provision).toHaveBeenCalledWith(host, {
+ forceFreshForWake: true,
+ targetSuspended: false,
+ })
+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(live().spec!.replicas).toBe(0)
+ expect(reconciler.getStatus(host.name)).toMatchObject({
+ deployed: true,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ })
+
it('reconciles an active Host twice through cache loss without changing its session path or template', async () => {
let cacheSynced = true
const host = makeStatelessHost({
@@ -1179,7 +1573,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
isCommunicationChannelCacheSynced: () => cacheSynced,
})
const host = makeStatelessHost({ status: suspendedStatus(4) })
- const live = persistHostDeployment(appsApi, host, reconciler.buildDeployment(host))
+ const applied = reconciler.buildDeployment(host)
+ markPersistedRuntimeTrusted(applied, host)
+ const live = persistHostDeployment(appsApi, host, applied)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
@@ -1470,7 +1866,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(provision).toHaveBeenCalledOnce()
})
- it('preserves runtime credentials during active cache loss without changing replicas', async () => {
+ it('refreshes only the GFS token during active cache loss without changing replicas', async () => {
let cacheSynced = true
const { reconciler, appsApi, customApi, networkingApi } = createReconciler({
isCommunicationChannelCacheSynced: () => cacheSynced,
@@ -1478,7 +1874,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
})
- const live = persistHostDeployment(appsApi, host, reconciler.buildDeployment(host))
+ const applied = reconciler.buildDeployment(host)
+ markPersistedRuntimeTrusted(applied, host)
+ const live = persistHostDeployment(appsApi, host, applied)
customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!
networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {
@@ -1492,7 +1890,11 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
await reconciler.reconcile(host)
expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()
- expect(provision).not.toHaveBeenCalled()
+ expect(provision).toHaveBeenCalledOnce()
+ expect(provision).toHaveBeenCalledWith(host, {
+ refreshGfsOnly: true,
+ targetSuspended: false,
+ })
const deployment = live()
expect(deployment.spec?.replicas).toBe(1)
expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index af0e94d7f..04d697d83 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -171,6 +171,7 @@ function createReconciler(
coreApi: asCoreApi(coreApi),
networkingApi: asNetworkingApi(networkingApi),
rbacApi: asRbacApi(rbacApi),
+ isCommunicationChannelCacheSynced: () => true,
infrastructureTelemetryReporter: options.infrastructureTelemetryReporter,
administrativeOutcomeReporter: options.administrativeOutcomeReporter,
newTelemetryOccurrenceId: options.newTelemetryOccurrenceId,