Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): hold stateless runtimes with verified scopes
Alfredo Lopez7655ab3d6dee
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 623c9384c..ca132875a 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.360",+ "version": "0.1.361","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.360",+ "version": "0.1.361","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 69bebc426..f772b7b60 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.360",+ "version": "0.1.361","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.tsindex 7f6f24ed4..4865643b9 100644--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts@@ -172,6 +172,8 @@ function runtimeSecretAnnotations(host = makeHost(), refreshBefore = '2999-01-01'clerum.io/gfs-token-host-uid': host.uid!,'clerum.io/runtime-token-refresh-before': refreshBefore,'clerum.io/gfs-token-refresh-before': '2999-01-01T00:00:00.000Z',+ 'clerum.io/runtime-token-has-channel-ingress': 'false',+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',}}@@ -233,6 +235,7 @@ function runtimeSecretReconciler(opts: {const reconciler = new HostReconciler(makeStubKc(), {coreApi: coreApi as unknown as k8s.CoreV1Api,appsApi: appsApi as unknown as k8s.AppsV1Api,+ isCommunicationChannelCacheSynced: () => true,})return { reconciler, coreApi, appsApi }}@@ -1256,6 +1259,7 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {const stubKc = makeStubKc()const reconciler = new HostReconciler(stubKc, {resolveContextMounts,+ isCommunicationChannelCacheSynced: () => true,// Stub APIs to capture the Deployment body and ack everything.coreApi: {readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(),diff --git a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.tsindex cde5f5035..37493ee32 100644--- a/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts+++ b/host-context-controller/src/hostReconciler.codexScopeProvenance.test.ts@@ -149,6 +149,7 @@ function createReconciler() {coreApi: asCoreApi(coreApi),networkingApi: asNetworkingApi(networkingApi),rbacApi: asRbacApi(rbacApi),+ isCommunicationChannelCacheSynced: () => true,})return { reconciler, appsApi, coreApi, networkingApi, rbacApi }}diff --git a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts b/host-context-controller/src/hostReconciler.grokHostRuntime.test.tsindex f9a5de272..9811f0da9 100644--- a/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts+++ b/host-context-controller/src/hostReconciler.grokHostRuntime.test.ts@@ -168,6 +168,7 @@ function createReconciler() {coreApi: asCoreApi(coreApi),networkingApi: asNetworkingApi(networkingApi),rbacApi: asRbacApi(rbacApi),+ isCommunicationChannelCacheSynced: () => true,})return { reconciler, appsApi, coreApi, networkingApi, rbacApi }}diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 87f5a4c79..ecb01b8c7 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -392,6 +392,8 @@ const RUNTIME_TOKEN_AUDIENCE_ANNOTATION = 'clerum.io/runtime-token-audience'const RUNTIME_TOKEN_SCHEMA_VERSION_ANNOTATION = 'clerum.io/runtime-token-schema-version'const RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION = 'clerum.io/runtime-token-bootstrap-state'const RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION = 'clerum.io/runtime-token-rollout-required'+const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'+const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'const RUNTIME_TOKEN_ISSUER = 'control-api'const RUNTIME_TOKEN_AUDIENCE = 'host-context-controller,workflow-approvals'// v3 extends the existing first-party access/refresh token material with the@@ -498,6 +500,7 @@ type HostSecretValidationResult =type BootstrapOptions = {forceFreshForWake?: booleantargetSuspended?: boolean+ refreshGfsOnly?: boolean}type RuntimeTokenProvision = {@@ -988,7 +991,7 @@ export class HostReconciler {}}- private isHccOwnedHostResource(+ private static isHccOwnedHostResource(resource: { metadata?: { labels?: Record<string, string> } },hostName: string): boolean {@@ -1013,7 +1016,7 @@ export class HostReconciler {throw error}- if (!this.isHccOwnedHostResource(resource, hostName)) {+ if (!HostReconciler.isHccOwnedHostResource(resource, hostName)) {log.warn('Skipping Host resource delete - not HCC-owned', { kind, name, host: hostName })return}@@ -1558,6 +1561,51 @@ export class HostReconciler {return (deployment?.status?.readyReplicas ?? 0) > 0}+ private static credentialRecordBelongsToHost(+ record: { metadata?: { labels?: Record<string, string> } },+ host: HostCRD+ ): boolean {+ return HostReconciler.isHccOwnedHostResource(record, host.name)+ }++ private static runtimeRecordIsComplete(+ record: Pick<k8s.V1Secret, 'data' | 'stringData'>+ ): boolean {+ return HostReconciler.runtimeTokenSecretData(record) !== null+ }++ private static retainedBoolean(record: k8s.V1Secret, key: string): boolean | null {+ const value = record.metadata?.annotations?.[key]+ return value === 'true' ? true : value === 'false' ? false : null+ }++ private static deploymentBelongsToHost(+ deployment: k8s.V1Deployment | null,+ host: HostCRD+ ): boolean {+ return (+ !!deployment &&+ HostReconciler.credentialRecordBelongsToHost(deployment, host) &&+ deployment.metadata?.name === host.name &&+ deployment.metadata?.namespace === host.namespace &&+ !!deployment.metadata?.uid &&+ !!deployment.metadata?.resourceVersion &&+ !deployment.metadata?.deletionTimestamp &&+ !!deployment.spec?.template?.spec &&+ !!host.uid &&+ deployment.metadata?.annotations?.[HOST_UID_ANNOTATION] === host.uid+ )+ }++ private static credentialRenewalWindowReached(+ record: k8s.V1Secret,+ nowMs: number,+ annotation: string+ ): boolean {+ const refreshBefore = Date.parse(record.metadata?.annotations?.[annotation] ?? '')+ return !Number.isFinite(refreshBefore) || nowMs >= refreshBefore+ }+/*** True when this reconcile will (re)start a pod that reads the* mcp-host-runtime-token Secret fresh at boot -- a missing Deployment, a@@ -1759,7 +1807,7 @@ export class HostReconciler {private async ensureMcpHostRuntimeTokenSecret(host: HostCRD,options: BootstrapOptions = {}- ): Promise<RuntimeTokenProvision> {+ ): Promise<RuntimeTokenProvision | null> {let lastErr: unknown = nullfor (let attempt = 1; attempt <= 3; attempt++) {try {@@ -1780,22 +1828,141 @@ export class HostReconciler {const existingRevision = existing? HostReconciler.runtimeTokenSecretRevisionFromSecret(existing): null+ const nowMs = Date.now()+ if (options.refreshGfsOnly) {+ const trustedDeployment =+ HostReconciler.deploymentBelongsToHost(deployment, host) &&+ (deployment?.spec?.replicas ?? 0) > 0 &&+ HostReconciler.deploymentReady(deployment)+ if (+ !existing ||+ !HostReconciler.credentialRecordBelongsToHost(existing, host) ||+ !host.uid ||+ existing.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION] !== host.uid ||+ !HostReconciler.runtimeRecordIsComplete(existing) ||+ !trustedDeployment+ ) {+ log.warn(+ 'deferring held-runtime credential renewal because runtime identity is not verified',+ {+ host: host.name,+ namespace: host.namespace,+ }+ )+ return null+ }+ if (+ !HostReconciler.credentialRenewalWindowReached(+ existing,+ nowMs,+ GFS_TOKEN_REFRESH_BEFORE_ANNOTATION+ )+ ) {+ this.recordGfsLifecycleEvidence(host, 'reused')+ return {+ revision:+ HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',+ scopeHash:+ existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',+ }+ }++ const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })+ const data = { ...(HostReconciler.runtimeTokenSecretData(existing) ?? {}) }+ data[MCP_HOST_GFS_TOKEN_SECRET_KEY] = Buffer.from(gfs.token).toString('base64')+ const gfsExpiresAtMs = nowMs + Math.max(0, gfs.expiresInSeconds) * 1000+ const gfsRefreshBeforeSec = HostReconciler.effectiveBootstrapRefreshBeforeSec(+ Math.max(0, gfs.expiresInSeconds)+ )+ const nextRevision = HostReconciler.runtimeTokenSecretRevision(data)+ await this.coreApi.replaceNamespacedSecret({+ name,+ namespace: host.namespace,+ body: {+ ...existing,+ data,+ metadata: {+ ...existing.metadata,+ annotations: {+ ...(existing.metadata?.annotations ?? {}),+ [GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:+ makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',+ [GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),+ [GFS_TOKEN_EXPIRES_AT_ANNOTATION]: new Date(gfsExpiresAtMs).toISOString(),+ [GFS_TOKEN_REFRESH_BEFORE_ANNOTATION]: new Date(+ gfsExpiresAtMs - gfsRefreshBeforeSec * 1000+ ).toISOString(),+ [RUNTIME_TOKEN_SECRET_REVISION_ANNOTATION]: nextRevision,+ },+ },+ },+ })+ this.recordGfsLifecycleEvidence(host, 'rotated')+ return {+ revision:+ HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision || '',+ scopeHash: existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',+ }+ }const bootstrapIsFresh =existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH- const nowMs = Date.now()- const hasChannelIngress = this.hasChannelIngress(host)+ const retainedAnnotations = existing?.metadata?.annotations ?? {}+ const cacheSynced = this.ccCacheSyncedFn()+ const retainedScopeIsTrusted =+ !!existing &&+ HostReconciler.credentialRecordBelongsToHost(existing, host) &&+ !!host.uid &&+ retainedAnnotations[GFS_TOKEN_HOST_UID_ANNOTATION] === host.uid+ const retainedChannelIngress =+ retainedScopeIsTrusted && existing+ ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION)+ : null+ const retainedFrontsOAuth =+ retainedScopeIsTrusted && existing+ ? HostReconciler.retainedBoolean(existing, RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION)+ : null+ const hasChannelIngress = cacheSynced+ ? this.hasChannelIngress(host)+ : retainedChannelIngress === true+ if (!cacheSynced && retainedChannelIngress === null) {+ log.warn(+ 'skipping runtime token mint during channel cache loss without retained scope observation',+ {+ host: host.name,+ namespace: host.namespace,+ observation: 'hasChannelIngress',+ }+ )+ return null+ }// Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and// codex projection into the mint-scope derive, the refresh decision, the// scope hash and the stored annotation, so the minted token and the drift// hash never diverge.- const frontsOAuthServer = await this.frontsOAuthServer(host)+ let observedFrontsOAuth: boolean+ if (cacheSynced) {+ observedFrontsOAuth = await this.frontsOAuthServer(host)+ } else {+ if (retainedFrontsOAuth === null) {+ log.warn(+ 'skipping runtime token mint during channel cache loss without retained OAuth observation',+ {+ host: host.name,+ namespace: host.namespace,+ observation: 'frontsOAuthServer',+ }+ )+ return null+ }+ observedFrontsOAuth = retainedFrontsOAuth+ }const projection = this.projectCodexForHost(host)const grokProjection = this.projectGrokForHost(host)const scopeHash = HostReconciler.runtimeTokenScopeHash(host,hasChannelIngress,- frontsOAuthServer,+ observedFrontsOAuth,projection,grokProjection)@@ -1810,7 +1977,7 @@ export class HostReconciler {existing,nowMs,hasChannelIngress,- frontsOAuthServer,+ observedFrontsOAuth,projection,grokProjection)@@ -1870,6 +2037,19 @@ export class HostReconciler {deploymentRevision !== '' &&deploymentRevision !== existingRevision)const annotationUpdates: Record<string, string> = {}+ if (cacheSynced) {+ const currentIngress =+ existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]+ if (currentIngress !== String(hasChannelIngress)) {+ annotationUpdates[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION] =+ String(hasChannelIngress)+ }+ const currentOAuth =+ existing.metadata?.annotations?.[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]+ if (currentOAuth !== String(observedFrontsOAuth)) {+ annotationUpdates[RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION] = String(observedFrontsOAuth)+ }+ }if (host.generation !== undefined &&existing.metadata?.annotations?.[GFS_TOKEN_HOST_GENERATION_ANNOTATION] !==@@ -1950,7 +2130,7 @@ export class HostReconciler {const tokens = await issueMcpHostRuntimeTokens(host.name,host.uid ?? '',- this.resolveEffectiveControlScopesForHost(host, frontsOAuthServer, hasChannelIngress)+ this.resolveEffectiveControlScopesForHost(host, observedFrontsOAuth, hasChannelIngress))const gfs = await mintHostGfsToken({ name: host.name, namespace: host.namespace })const body = buildMcpHostRuntimeTokenSecret(@@ -1978,13 +2158,15 @@ export class HostReconciler {nowMs,gfs.expiresInSeconds,hasChannelIngress,- frontsOAuthServer,+ observedFrontsOAuth,existing?.metadata?.annotations?.[GFS_TOKEN_HOST_UID_ANNOTATION],projection,grokProjection),[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',+ [RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),+ [RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION]: String(observedFrontsOAuth),},}@@ -2055,7 +2237,7 @@ export class HostReconciler {private async provisionRuntimeTokenRevision(host: HostCRD,options: BootstrapOptions- ): Promise<RuntimeTokenProvision> {+ ): Promise<RuntimeTokenProvision | null> {try {return await this.ensureMcpHostRuntimeTokenSecret(host, options)} catch (err) {@@ -3653,7 +3835,7 @@ export class HostReconciler {if (!observedDeployment) return nullconst existing = observedDeploymentif (- !this.isHccOwnedHostResource(existing, host.name) ||+ !HostReconciler.isHccOwnedHostResource(existing, host.name) ||existing.metadata?.name !== host.name ||existing.metadata?.namespace !== host.namespace ||!existing.metadata?.uid ||@@ -4800,17 +4982,53 @@ export class HostReconciler {return false}}- const holdAppliedRuntime = async (allowScaleUp: boolean): Promise<void> => {- if (this.ccCacheSyncedFn()) {- await this.provisionRuntimeTokenRevision(host, {- forceFreshForWake: allowScaleUp,+ const holdAppliedRuntime = async (wakeRequested: boolean): Promise<void> => {+ const liveDeployment = await this.readHostDeploymentOrNull(host)+ const deploymentIdentityIsTrusted =+ !liveDeployment || HostReconciler.deploymentBelongsToHost(liveDeployment, host)+ const runtimeIsReady =+ !!liveDeployment &&+ HostReconciler.deploymentBelongsToHost(liveDeployment, host) &&+ HostReconciler.deploymentReady(liveDeployment)+ const scaleUpForRecovery =+ lifecycle.effective.state === 'active' &&+ (!liveDeployment || (liveDeployment.spec?.replicas ?? 1) === 0)+ const allowScaleUp = deploymentIdentityIsTrusted && (wakeRequested || scaleUpForRecovery)+ let provisioningRequired = false+ if (allowScaleUp) {+ // A pod starting from zero will consume bootstrap credentials. Mint and+ // persist them before the replica-only update; a failed mint aborts here.+ provisioningRequired = true+ } else if (lifecycle.effective.state === 'active' && runtimeIsReady) {+ const provision = await this.provisionRuntimeTokenRevision(host, {+ targetSuspended: false,+ refreshGfsOnly: true,+ })+ if (!provision) {+ this.setStatus(host.name, {+ deployed: true,+ ready: false,+ message: 'Held runtime credential renewal is not safely available',+ })+ return+ }+ revalidateHostMutationBoundary()+ }+ if (provisioningRequired) {+ const provision = await this.provisionRuntimeTokenRevision(host, {+ forceFreshForWake: true,targetSuspended: lifecycle.effective.state === 'suspended',})+ if (!provision) {+ this.setStatus(host.name, {+ deployed: !!liveDeployment,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })+ return+ }revalidateHostMutationBoundary()}- // The channel inventory is incomplete while its cache is unsynced. Defer- // scope-sensitive token minting until the next authoritative reconcile so- // a missing channel count cannot narrow the running Host's token scopes.const applied = await this.ensureDeployment(host,mounts,@@ -4961,6 +5179,23 @@ export class HostReconciler {forceFreshForWake,targetSuspended: lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})+ if (!runtimeTokenProvision) {+ const liveDeployment = await this.readHostDeploymentOrNull(host)+ this.setStatus(host.name, {+ deployed: !!liveDeployment,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })+ return+ }+ const requireRuntimeTokenProvision = (): RuntimeTokenProvision => {+ if (!runtimeTokenProvision) {+ const error = new Error('Waiting for authoritative scope observation')+ error.name = 'RuntimeScopeObservationUnavailableError'+ throw error+ }+ return runtimeTokenProvision+ }revalidateHostMutationBoundary()// replaceWithConflictRetry may wait and re-read after a 409, so each body is@@ -4984,19 +5219,24 @@ export class HostReconciler {for (let attempt = 1; attempt <= 3; attempt++) {revalidateHostMutationBoundary()const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))- if (runtimeTokenProvision.scopeHash === currentScopeHash) return+ if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) returnruntimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {forceFreshForWake: false,targetSuspended:lifecycle.effective.stateless && lifecycle.effective.state === 'suspended',})+ if (!runtimeTokenProvision) {+ const error = new Error('Waiting for authoritative scope observation')+ error.name = 'RuntimeScopeObservationUnavailableError'+ throw error+ }revalidateHostMutationBoundary()const postProvisionScopeHash = this.runtimeScopeHashFor(host,await this.frontsOAuthServer(host))- if (runtimeTokenProvision.scopeHash === postProvisionScopeHash) return+ if (requireRuntimeTokenProvision().scopeHash === postProvisionScopeHash) returnlog.warn('CommunicationChannel scope contract changed during token provisioning', {host: host.name,@@ -5016,7 +5256,7 @@ export class HostReconciler {if (beforeScope.suspensionBlocked) {return {lifecycle: beforeScope,- runtimeTokenRevision: runtimeTokenProvision.revision,+ runtimeTokenRevision: requireRuntimeTokenProvision().revision,grokExecutionEnabled: this.hostDerivesGrokExecution(host),}}@@ -5031,11 +5271,11 @@ export class HostReconciler {) {continue}- if (runtimeTokenProvision.scopeHash === currentScopeHash) {+ if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) {revalidateHostMutationBoundary()return {lifecycle: effective,- runtimeTokenRevision: runtimeTokenProvision.revision,+ runtimeTokenRevision: requireRuntimeTokenProvision().revision,grokExecutionEnabled: this.hostDerivesGrokExecution(host),}}@@ -5051,14 +5291,28 @@ export class HostReconciler {}revalidateHostMutationBoundary()- const deploymentApplied = await this.ensureDeployment(- host,- mounts,- runtimeTokenProvision.revision,- lifecycle.effective,- resolveDeploymentState,- revalidateHostMutationBoundary- )+ let deploymentApplied: boolean+ try {+ deploymentApplied = await this.ensureDeployment(+ host,+ mounts,+ runtimeTokenProvision.revision,+ lifecycle.effective,+ resolveDeploymentState,+ revalidateHostMutationBoundary+ )+ } catch (error) {+ if (error instanceof Error && error.name === 'RuntimeScopeObservationUnavailableError') {+ const liveDeployment = await this.readHostDeploymentOrNull(host)+ this.setStatus(host.name, {+ deployed: !!liveDeployment,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })+ return+ }+ throw error+ }revalidateHostMutationBoundary()if (!deploymentApplied) {diff --git a/host-context-controller/src/statelessLifecycle.types.ts b/host-context-controller/src/statelessLifecycle.types.tsindex 6f7b098fe..ebaa1410f 100644--- a/host-context-controller/src/statelessLifecycle.types.ts+++ b/host-context-controller/src/statelessLifecycle.types.ts@@ -17,7 +17,7 @@ export interface EffectiveHostLifecycle {state: HostLifecycleState/** Unknown channel authority: preserve the entire applied Deployment template. */suspensionBlocked?: boolean- /** A fresh-read-verified wake may scale the held Deployment without a template change. */+ /** An active recovery or fresh-read-verified wake may scale the held Deployment without a template change. */allowScaleUpDuringHold?: boolean}diff --git a/host-context-controller/src/statelessLifecycleExecutor.ts b/host-context-controller/src/statelessLifecycleExecutor.tsindex 4647ce538..f556dced4 100644--- a/host-context-controller/src/statelessLifecycleExecutor.ts+++ b/host-context-controller/src/statelessLifecycleExecutor.ts@@ -231,7 +231,11 @@ export class StatelessLifecycleExecutor {return { stateless: false, state: 'active' }}if (!this.isCommunicationChannelCacheSynced()) {- return { stateless: true, state: 'active', suspensionBlocked: true }+ return {+ stateless: true,+ state: host.status?.lifecycle?.state ?? 'active',+ suspensionBlocked: true,+ }}return { stateless: true, state: host.status?.lifecycle?.state ?? 'active' }}@@ -1790,8 +1794,8 @@ export class StatelessLifecycleExecutor {assessment: HostLifecycleAssessment): Promise<HostLifecycleAssessment | null> {if (assessment.effective.suspensionBlocked) {- // Cache authority loss intentionally forces active even when the fresh- // durable state is suspended; there is no scale-down to validate.+ // Cache authority loss blocks lifecycle-driven scale-down; preserve the+ // durable state for accurate reporting without mutating the Deployment.return assessment}if (!assessment.effective.stateless) {diff --git a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.tsindex b0e5605c3..daf609c93 100644--- a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts+++ b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts@@ -205,48 +205,28 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synce})})-// ── B2: unsynced cache + existing Deployment has replicas=1 → must preserve ──-describe('reconcile() — B2 integration: replica preservation when CC cache is unsynced (409 path)', () => {- it('preserves replicas=1 on replaceNamespacedDeployment body when cache is unsynced and existing Deployment is live', async () => {- // Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced).- // The existing live Deployment has replicas=1 (channel-reader is running).- // reconcile() MUST NOT scale it to 0 — that would drop inbound messages.+// ── B2: unsynced scope with no retained observation must defer Deployment ──+describe('reconcile() — B2 integration: Deployment is deferred without an authoritative scope observation', () => {+ it('does not mutate Deployment when the CC cache is unsynced and scope was not retained', async () => {+ // Scenario: HCC restarts mid-flight. CC cache is still loading (unsynced),+ // and the runtime Secret has no trusted retained scope observation. The+ // reconciler must wait before changing the runtime Deployment.const { reconciler, appsApi } = createReconciler()- // Force the 409 → read-existing → replace path ONLY for the channels namespace- // channel-reader Deployment. The mcp-host Deployment (mcp-host ns) must succeed- // so that reconcile() progresses to the channel-reader step.+ // A Deployment create would expose the runtime to credentials minted from+ // an incomplete channel inventory; make any attempted create fail loudly.appsApi.createNamespacedDeployment.mockImplementation(({ body }: { namespace: string; body: k8s.V1Deployment }) => {const dep = body as k8s.V1Deployment- if (dep.metadata?.namespace === 'channels') {- return Promise.reject({ code: 409 })- }- // mcp-host Deployment (mcp-host ns) and everything else succeeds normally- return Promise.resolve({})+ return Promise.reject(+ new Error(`Unexpected Deployment create in ${dep.metadata?.namespace}`)+ )})- // readNamespacedDeployment is called for both the host (name=alpha-host, ns=mcp-host)- // and the channel-reader (name=channel-reader-alpha-host, ns=channels).- // Return a "live replicas=1" response for the channel-reader read; for the host- // deployment read, return HCC-owned labels so reconcile() continues normally.+ // Return an existing live, HCC-owned runtime Deployment. It must remain+ // untouched while the reconciler waits for authoritative channel scope.appsApi.readNamespacedDeployment.mockImplementation(({ name, namespace }: { name: string; namespace: string }) => {- if (name === 'channel-reader-alpha-host') {- return Promise.resolve({- metadata: {- name: 'channel-reader-alpha-host',- namespace: 'channels',- resourceVersion: '77',- labels: {- 'clerum.io/host': 'alpha-host',- 'clerum.io/managed-by': 'host-context-controller',- },- },- spec: { replicas: 1 },- })- }- // Default: HCC-owned host Deployment (for readiness check)return Promise.resolve({status: { readyReplicas: 1 },metadata: {@@ -270,15 +250,13 @@ describe('reconcile() — B2 integration: replica preservation when CC cache isawait reconciler.reconcile(makeHost())- // The replace call body must carry replicas=1, not 0- expect(- appsApi.replaceNamespacedDeployment.mock.calls.filter(- ([request]) => request.namespace === 'channels'- )- ).toHaveLength(1)+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()- const replaceBody = getChannelReaderReplaceBody(appsApi)- expect(replaceBody.spec?.replicas).toBe(1)+ expect(reconciler.getStatus('alpha-host')).toMatchObject({+ deployed: true,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })})})diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 4661a4c01..76e5a8980 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -5,11 +5,18 @@ import { mintHostGfsToken } from '../src/gfsHostBinding'import {type EffectiveHostLifecycle,HostReconciler,+ OAUTH_USER_TOKEN_SCOPE,type ResolvedSfsMount,} from '../src/hostReconciler'import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'import { HostContextLogger } from '../src/logger'import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'+import {+ MCP_HOST_GFS_TOKEN_SECRET_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,+} from '../src/secretFactory'import { HostCRD, HostCrdStatus } from '../src/types'import {type MockAppsApi,@@ -195,6 +202,98 @@ function runtimeTokenProvision(host: HostCRD, hasChannelIngress = false) {}}+function runtimeCredentialRecord(+ host: HostCRD,+ options: {+ gfsRefreshBefore?: string+ frontsOAuthServer?: string+ hasChannelIngress?: string+ hostUid?: string+ managedByHost?: boolean+ } = {}+) {+ const data = {+ [MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]: Buffer.from('access-value').toString('base64'),+ [MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]: Buffer.from('refresh-value').toString('base64'),+ [MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]: Buffer.from('control-value').toString('base64'),+ [MCP_HOST_GFS_TOKEN_SECRET_KEY]: Buffer.from('gfs-value').toString('base64'),+ }+ const revision = (+ HostReconciler as unknown as {+ runtimeTokenSecretRevision(data: Record<string, string>): string+ }+ ).runtimeTokenSecretRevision(data)+ return {+ metadata: {+ name: `host-${host.name}-mcp-host-runtime-tokens`,+ namespace: host.namespace,+ resourceVersion: '88',+ labels:+ options.managedByHost === false+ ? { 'clerum.io/managed-by': 'someone-else' }+ : {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/host': host.name,+ },+ annotations: {+ 'clerum.io/gfs-token-host-uid': options.hostUid ?? host.uid ?? '',+ 'clerum.io/runtime-token-secret-revision': revision,+ 'clerum.io/runtime-token-has-channel-ingress': options.hasChannelIngress ?? 'false',+ 'clerum.io/runtime-token-fronts-oauth-server': options.frontsOAuthServer ?? 'false',+ 'clerum.io/gfs-token-refresh-before':+ options.gfsRefreshBefore ?? '2999-01-01T00:00:00.000Z',+ },+ },+ data,+ } as unknown as k8s.V1Secret+}++function trustedRuntimeDeployment(+ reconciler: HostReconciler,+ host: HostCRD,+ options: { readyReplicas?: number; replicas?: number } = {}+) {+ const deployment = reconciler.buildDeployment(host)+ deployment.metadata = {+ ...deployment.metadata,+ uid: `deployment-${host.name}`,+ resourceVersion: '91',+ labels: {+ ...(deployment.metadata?.labels ?? {}),+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/host': host.name,+ },+ annotations: {+ ...(deployment.metadata?.annotations ?? {}),+ 'clerum.io/host-uid': host.uid ?? '',+ },+ }+ deployment.spec = { ...deployment.spec, replicas: options.replicas ?? 1 }+ deployment.status = { readyReplicas: options.readyReplicas ?? 1 }+ return deployment+}++function markPersistedRuntimeTrusted(+ deployment: k8s.V1Deployment,+ host: HostCRD,+ readyReplicas = 1+): k8s.V1Deployment {+ deployment.metadata = {+ ...deployment.metadata,+ labels: {+ ...(deployment.metadata?.labels ?? {}),+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/host': host.name,+ },+ annotations: {+ ...(deployment.metadata?.annotations ?? {}),+ 'clerum.io/host-uid': host.uid ?? '',+ },+ }+ deployment.status = { readyReplicas }+ return deployment+}+/** Model persisted API state so a loss-of-authority test cannot invent a template. */function persistHostDeployment(appsApi: MockAppsApi, host: HostCRD, initial: k8s.V1Deployment) {let live = structuredClone(initial)@@ -313,12 +412,12 @@ describe('HostReconciler stateless lifecycle — buildDeployment replicas', () =expect(dep.spec?.replicas).toBe(1)})- it('forces stateless+suspended to replicas=1 while the channel cache is unsynced', () => {+ it('preserves stateless+suspended replicas while the channel cache is unsynced', () => {const { reconciler } = createReconciler({isCommunicationChannelCacheSynced: () => false,})const dep = reconciler.buildDeployment(makeStatelessHost({ status: suspendedStatus() }))- expect(dep.spec?.replicas).toBe(1)+ expect(dep.spec?.replicas).toBe(0)expect(dep.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')})@@ -811,6 +910,10 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {countCommunicationChannels: () => 1,isCommunicationChannelCacheSynced: () => false,})+ const read = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ request.name === host.name ? Promise.reject({ code: 404 }) : read(request)+ )customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))const applied = createReconciler().reconciler.buildDeployment(host)applied.metadata = { ...applied.metadata, uid: 'deployment-uid', resourceVersion: '74' }@@ -971,7 +1074,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()})- it('preserves runtime credentials and the existing template during cache loss', async () => {+ it('refreshes only the GFS token for a running held Host during cache loss', async () => {const host = makeStatelessHost({status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },})@@ -981,12 +1084,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))const applied = reconciler.buildDeployment(host)applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }+ markPersistedRuntimeTrusted(applied, host)const live = persistHostDeployment(appsApi, host, applied)const provision = vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host))await reconciler.reconcile(host)- expect(provision).not.toHaveBeenCalled()+ expect(provision).toHaveBeenCalledOnce()+ expect(provision).toHaveBeenCalledWith(host, {+ refreshGfsOnly: true,+ targetSuspended: false,+ })expect(live().spec!.replicas).toBe(1)expect(live().spec!.template).toEqual(applied.spec!.template)expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(@@ -1032,7 +1140,14 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host))await reconciler.reconcile(host)- expect(provision).not.toHaveBeenCalled()+ expect(provision).toHaveBeenCalledOnce()+ expect(provision).toHaveBeenCalledWith(host, {+ forceFreshForWake: true,+ targetSuspended: false,+ })+ expect(provision.mock.invocationCallOrder[0]).toBeLessThan(+ appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!+ )expect(appsApi.patchNamespacedDeployment).not.toHaveBeenCalled()expect(live().spec!.replicas).toBe(1)expect(live().spec!.template).toEqual(applied.spec!.template)@@ -1088,11 +1203,26 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {it('preserves an independently confirmed desktop rejection while the channel cache is unknown', async () => {const host = makeStatelessHost({ spec: { desktop: { browser: true } } })- const { reconciler, appsApi, customApi } = createReconciler({+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({countCommunicationChannels: () => 1,isCommunicationChannelCacheSynced: () => false,})customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ coreApi.readNamespacedSecret.mockResolvedValue({+ metadata: {+ resourceVersion: '1',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/host': host.name,+ },+ annotations: {+ 'clerum.io/gfs-token-host-uid': host.uid,+ 'clerum.io/runtime-token-has-channel-ingress': 'true',+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',+ },+ },+ data: {},+ } as any)await reconciler.reconcile(host)const condition = rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!)expect(condition.reason).toBe('DesktopEnabled')@@ -1104,6 +1234,270 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {).toBe(false)})+ it('defers a confirmed desktop rejection when no trusted scope observation exists', async () => {+ const host = makeStatelessHost({ spec: { desktop: { browser: true } } })+ const { reconciler, appsApi, customApi } = createReconciler({+ countCommunicationChannels: () => 1,+ isCommunicationChannelCacheSynced: () => false,+ })+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ request.name === host.name ? Promise.reject({ code: 404 }) : readDeployment(request)+ )+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))++ await reconciler.reconcile(host)++ expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(+ 'DesktopEnabled'+ )+ expect(+ [+ ...appsApi.createNamespacedDeployment.mock.calls,+ ...appsApi.replaceNamespacedDeployment.mock.calls,+ ].filter(+ ([request]) => (request as { body?: k8s.V1Deployment }).body?.metadata?.name === host.name+ )+ ).toHaveLength(0)+ expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })+ })++ it('does not let a retained OAuth observation override a failed live probe', async () => {+ const host = makeStatelessHost()+ const { reconciler, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => true,+ })+ coreApi.readNamespacedSecret.mockResolvedValue(+ runtimeCredentialRecord(host, { frontsOAuthServer: 'true' })+ )+ reconciler.setHostFrontsOAuthServer(async () => {+ throw new Error('oauth lookup unavailable')+ })+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)++ expect(provision).not.toBeNull()+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).not.toContain(+ OAUTH_USER_TOKEN_SCOPE+ )+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret+ expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(+ 'false'+ )+ })++ it('uses only retained scope observations while channel authority is unavailable', async () => {+ const host = makeStatelessHost()+ const { reconciler, coreApi } = createReconciler({+ countCommunicationChannels: () => 0,+ isCommunicationChannelCacheSynced: () => false,+ })+ coreApi.readNamespacedSecret.mockResolvedValue(+ runtimeCredentialRecord(host, {+ frontsOAuthServer: 'true',+ hasChannelIngress: 'true',+ })+ )+ const oauthProbe = vi+ .spyOn(reconciler as any, 'frontsOAuthServer')+ .mockImplementation(async () => {+ throw new Error('live lookup must not run')+ })+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)++ expect(provision).not.toBeNull()+ expect(oauthProbe).not.toHaveBeenCalled()+ expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).toContain(+ OAUTH_USER_TOKEN_SCOPE+ )+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret+ expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(+ 'true'+ )+ })++ it('does not renew held-runtime GFS before its refresh window', async () => {+ const host = makeStatelessHost()+ const { reconciler, appsApi, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ coreApi.readNamespacedSecret.mockResolvedValue(runtimeCredentialRecord(host))+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))+ vi.mocked(mintHostGfsToken).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {+ refreshGfsOnly: true,+ })++ expect(provision).not.toBeNull()+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()+ })++ it('renews only held-runtime GFS at the refresh boundary', async () => {+ const host = makeStatelessHost()+ const { reconciler, appsApi, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ const record = runtimeCredentialRecord(host, {+ gfsRefreshBefore: '2000-01-01T00:00:00.000Z',+ })+ coreApi.readNamespacedSecret.mockResolvedValue(record)+ const deployment = trustedRuntimeDeployment(reconciler, host)+ deployment.spec!.template!.metadata!.annotations = {+ ...deployment.spec!.template!.metadata!.annotations,+ 'clerum.io/runtime-token-revision': 'applied-runtime-revision',+ }+ appsApi.readNamespacedDeployment.mockResolvedValue(deployment)+ vi.mocked(mintHostGfsToken).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {+ refreshGfsOnly: true,+ })++ expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()+ expect(provision.revision).toBe('applied-runtime-revision')+ const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]).toBe(+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY]+ )+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]).toBe(+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY]+ )+ expect(write.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]).toBe(+ record.data?.[MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY]+ )+ expect(write.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(+ record.data?.[MCP_HOST_GFS_TOKEN_SECRET_KEY]+ )+ })++ it('never falls back to full issuance from a closed held-runtime GFS renewal', async () => {+ const host = makeStatelessHost()+ const { reconciler, appsApi, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()+ vi.mocked(mintHostGfsToken).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {+ refreshGfsOnly: true,+ })++ expect(provision).toBeNull()+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()+ })++ it('does not treat a requested but unready held runtime as ready for GFS renewal', async () => {+ const host = makeStatelessHost()+ const { reconciler, appsApi, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ coreApi.readNamespacedSecret.mockResolvedValue(+ runtimeCredentialRecord(host, { gfsRefreshBefore: '2000-01-01T00:00:00.000Z' })+ )+ appsApi.readNamespacedDeployment.mockResolvedValue(+ trustedRuntimeDeployment(reconciler, host, { readyReplicas: 0 })+ )+ vi.mocked(mintHostGfsToken).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {+ refreshGfsOnly: true,+ })++ expect(provision).toBeNull()+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()+ })++ it('does not reassign a prior Host UID during held-runtime GFS renewal', async () => {+ const host = makeStatelessHost()+ const { reconciler, appsApi, coreApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ coreApi.readNamespacedSecret.mockResolvedValue(+ runtimeCredentialRecord(host, { hostUid: 'prior-host-uid' })+ )+ appsApi.readNamespacedDeployment.mockResolvedValue(trustedRuntimeDeployment(reconciler, host))+ vi.mocked(mintHostGfsToken).mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host, {+ refreshGfsOnly: true,+ })++ expect(provision).toBeNull()+ expect(vi.mocked(mintHostGfsToken)).not.toHaveBeenCalled()+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()+ })++ it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {+ const host = makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },+ })+ const { reconciler, appsApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ const applied = reconciler.buildDeployment(host)+ applied.spec!.replicas = 0+ applied.spec!.template.metadata!.annotations = { 'example.org/applied': 'keep-exactly' }+ const live = persistHostDeployment(appsApi, host, applied)+ const provision = vi+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')+ .mockResolvedValue(runtimeTokenProvision(host))++ await reconciler.reconcile(host)++ expect(provision).toHaveBeenCalledOnce()+ expect(provision).toHaveBeenCalledWith(host, {+ forceFreshForWake: true,+ targetSuspended: false,+ })+ expect(provision.mock.invocationCallOrder[0]).toBeLessThan(+ appsApi.replaceNamespacedDeployment.mock.invocationCallOrder.at(-1)!+ )+ expect(live().spec!.replicas).toBe(1)+ expect(live().spec!.template).toEqual(applied.spec!.template)+ })++ it('keeps an active held Deployment at zero when credential minting is unavailable', async () => {+ const host = makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },+ })+ const { reconciler, appsApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ const applied = reconciler.buildDeployment(host)+ applied.spec!.replicas = 0+ markPersistedRuntimeTrusted(applied, host, 0)+ const live = persistHostDeployment(appsApi, host, applied)+ appsApi.replaceNamespacedDeployment.mockClear()+ const provision = vi+ .spyOn(reconciler as any, 'provisionRuntimeTokenRevision')+ .mockResolvedValue(null)++ await reconciler.reconcile(host)++ expect(provision).toHaveBeenCalledOnce()+ expect(provision).toHaveBeenCalledWith(host, {+ forceFreshForWake: true,+ targetSuspended: false,+ })+ expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()+ expect(live().spec!.replicas).toBe(0)+ expect(reconciler.getStatus(host.name)).toMatchObject({+ deployed: true,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })+ })+it('reconciles an active Host twice through cache loss without changing its session path or template', async () => {let cacheSynced = trueconst host = makeStatelessHost({@@ -1179,7 +1573,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {isCommunicationChannelCacheSynced: () => cacheSynced,})const host = makeStatelessHost({ status: suspendedStatus(4) })- const live = persistHostDeployment(appsApi, host, reconciler.buildDeployment(host))+ const applied = reconciler.buildDeployment(host)+ markPersistedRuntimeTrusted(applied, host)+ const live = persistHostDeployment(appsApi, host, applied)customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {@@ -1470,7 +1866,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(provision).toHaveBeenCalledOnce()})- it('preserves runtime credentials during active cache loss without changing replicas', async () => {+ it('refreshes only the GFS token during active cache loss without changing replicas', async () => {let cacheSynced = trueconst { reconciler, appsApi, customApi, networkingApi } = createReconciler({isCommunicationChannelCacheSynced: () => cacheSynced,@@ -1478,7 +1874,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {const host = makeStatelessHost({status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },})- const live = persistHostDeployment(appsApi, host, reconciler.buildDeployment(host))+ const applied = reconciler.buildDeployment(host)+ markPersistedRuntimeTrusted(applied, host)+ const live = persistHostDeployment(appsApi, host, applied)customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))const readPolicy = networkingApi.readNamespacedNetworkPolicy.getMockImplementation()!networkingApi.readNamespacedNetworkPolicy.mockImplementation(async request => {@@ -1492,7 +1890,11 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {await reconciler.reconcile(host)expect(networkingApi.readNamespacedNetworkPolicy).toHaveBeenCalled()- expect(provision).not.toHaveBeenCalled()+ expect(provision).toHaveBeenCalledOnce()+ expect(provision).toHaveBeenCalledWith(host, {+ refreshGfsOnly: true,+ targetSuspended: false,+ })const deployment = live()expect(deployment.spec?.replicas).toBe(1)expect(envValue(deployment, 'CLERUM_STATELESS_LIFECYCLE')).toBe('true')diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex af0e94d7f..04d697d83 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -171,6 +171,7 @@ function createReconciler(coreApi: asCoreApi(coreApi),networkingApi: asNetworkingApi(networkingApi),rbacApi: asRbacApi(rbacApi),+ isCommunicationChannelCacheSynced: () => true,infrastructureTelemetryReporter: options.infrastructureTelemetryReporter,administrativeOutcomeReporter: options.administrativeOutcomeReporter,newTelemetryOccurrenceId: options.newTelemetryOccurrenceId,