Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): separate bootstrap intent from runtime maintenance
Alfredo Lopez7dd1547ebf78
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex db02055f0..938e400a2 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.376",+ "version": "0.1.377","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.376",+ "version": "0.1.377","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex eb45e90ac..ef4a910fe 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.376",+ "version": "0.1.377","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.tsindex 1db6912e7..7afe1ba4a 100644--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts@@ -184,7 +184,18 @@ function deploymentWithRevision(replicas = 1): k8s.V1Deployment {return {- metadata: { name: 'team-mission', namespace: 'mcp-host', generation: 1 },+ metadata: {+ name: 'team-mission',+ namespace: 'mcp-host',+ generation: 1,+ uid: 'team-mission-deployment-uid',+ resourceVersion: '1',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/host': 'team-mission',+ },+ annotations: { 'clerum.io/host-uid': 'team-mission-uid' },+ },spec: {replicas,template: {@@ -371,7 +382,15 @@ describe('HostReconciler runtime credential Secret revision', () => {expect(result.revision).toBe(currentSecretRevision)expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()- expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()+ // Applied-target bookkeeping is one CAS write, with no credential change.+ expect(coreApi.replaceNamespacedSecret).toHaveBeenCalledOnce()+ const applied = replacedRuntimeSecret(coreApi)+ expect(applied.metadata?.annotations?.[BOOTSTRAP_STATE_KEY]).toBe('fresh')+ expect(applied.data).toEqual(secret.data)+ Object.assign(secret, applied, {+ metadata: { ...applied.metadata, resourceVersion: '20' },+ })+ coreApi.replaceNamespacedSecret.mockClear()deployment.status!.readyReplicas = 1await (diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex bc2597038..78fcce468 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -33,6 +33,7 @@ import {WFC_APP_LABEL,sharedFileSystemHash,} from './k8s/sharedFileSystemFactory'+import { kubernetesQuantitiesEqual } from './kubernetesQuantity'import {ALLOWED_MODELS_CONFIGMAP_NAME,parseAllowedModelsSnapshot,@@ -149,6 +150,13 @@ class HostMutationIdentityChangedError extends Error {}}+class RuntimeBootstrapPreparationUnavailableError extends Error {+ constructor() {+ super('Waiting for a prepared runtime bootstrap before changing resources')+ this.name = 'RuntimeBootstrapPreparationUnavailableError'+ }+}+class HostMutationSpecRevisionChangedError extends Error {constructor(action: string, hostName: string) {super(`Host spec generation changed before ${action} admission for "${hostName}"`)@@ -396,6 +404,8 @@ const RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION ='clerum.io/runtime-token-bootstrap-deployment-uid'const RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION ='clerum.io/runtime-token-bootstrap-applied-revision'+const RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION =+ 'clerum.io/runtime-token-bootstrap-ready-revision'const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'const RUNTIME_TOKEN_ISSUER = 'control-api'@@ -503,6 +513,7 @@ type HostSecretValidationResult =type BootstrapOptions = {forceFreshForWake?: boolean+ forceFreshForDeploymentChange?: booleantargetSuspended?: booleanprepareForResourceChange?: booleanrefreshGfsOnly?: boolean@@ -512,6 +523,8 @@ type BootstrapOptions = {type RuntimeTokenProvision = {revision: stringscopeHash: string+ deploymentUid?: string+ freshBootstrapPrepared?: boolean}// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).@@ -520,6 +533,7 @@ type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'type DeploymentMutationState = {lifecycle: EffectiveHostLifecycleruntimeTokenRevision: string+ freshBootstrapPrepared?: boolean// Captured in the same synchronous step as the stable scope-hash check so// the pod's Grok factory switch and the minted llm:grok:execute scope agree.grokExecutionEnabled: boolean@@ -1589,11 +1603,24 @@ export class HostReconciler {host.spec.desktop?.browser || host.spec.desktop?.x11? config.desktopResources: config.hostResources- // Match the Deployment comparator's exact quantity values, ignoring map order only.- return isDeepStrictEqual(- canonicalizeValue({ requests: resources?.requests, limits: resources?.limits }),- canonicalizeValue(desired)- )+ let matches = true+ for (const kind of ['requests', 'limits'] as const) {+ const actual = resources?.[kind]+ const expected = desired[kind]+ if (!isDeepStrictEqual(Object.keys(actual ?? {}).sort(), Object.keys(expected).sort())) {+ matches = false+ }+ for (const [key, value] of Object.entries(expected)) {+ // Validate desired quantities even when a field is absent. A lexical+ // PUT stays governed by the exact Deployment comparator, but equal+ // Kubernetes amounts do not start a new Pod or need new credentials.+ kubernetesQuantitiesEqual(value, value)+ if (actual?.[key] === undefined || !kubernetesQuantitiesEqual(actual[key], value)) {+ matches = false+ }+ }+ }+ return matches}/**@@ -1626,6 +1653,64 @@ export class HostReconciler {)}+ private static bootstrapAppliedToDeployment(+ record: k8s.V1Secret,+ deployment: k8s.V1Deployment | null,+ contentRevision: string+ ): boolean {+ if (HostReconciler.bootstrapBindingMatchesDeployment(record, deployment)) return true+ // An existing target binding must never be adopted by a same-name/new-UID+ // Deployment. Older unbound records can be bound by their applied revision.+ if (record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]) {+ return false+ }+ return (+ contentRevision !== '' &&+ HostReconciler.deploymentRuntimeTokenRevision(deployment) === contentRevision+ )+ }++ private static bootstrapObservationUpdates(+ host: HostCRD,+ record: k8s.V1Secret,+ deployment: k8s.V1Deployment | null,+ contentRevision: string+ ): Record<string, string> {+ if (+ record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==+ RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||+ !HostReconciler.deploymentBelongsToHost(deployment, host) ||+ !HostReconciler.bootstrapAppliedToDeployment(record, deployment, contentRevision)+ )+ return {}+ const current = record.metadata?.annotations ?? {}+ const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)+ const desired: Record<string, string> = {+ [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: revision,+ // Application is distinct from consumption: GFS content may change+ // while the same bootstrap waits for strict acknowledgement.+ [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',+ }+ if (HostReconciler.deploymentBootstrapConverged(deployment)) {+ desired[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] = RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED+ } else if (+ (deployment!.spec?.replicas ?? 1) > 0 &&+ (deployment!.metadata!.generation ?? 0) > 0 &&+ (deployment!.status?.observedGeneration ?? 0) >= deployment!.metadata!.generation! &&+ (deployment!.status?.updatedReplicas ?? 0) > 0 &&+ HostReconciler.deploymentReady(deployment)+ ) {+ // Aggregate Ready is only possible-use evidence, not an acknowledgement.+ // Persist it for this UID/revision so a later loss of Ready can recover+ // once; uninterrupted NotReady startup never acquires this observation.+ desired[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] = revision+ }+ return Object.fromEntries(+ Object.entries(desired).filter(([key, value]) => current[key] !== value)+ )+ }+private static credentialRecordBelongsToHost(record: { metadata?: { labels?: Record<string, string> } },host: HostCRD@@ -1647,7 +1732,7 @@ export class HostReconciler {private static deploymentBelongsToHost(deployment: k8s.V1Deployment | null,host: HostCRD- ): boolean {+ ): deployment is k8s.V1Deployment {return (!!deployment &&HostReconciler.credentialRecordBelongsToHost(deployment, host) &&@@ -1699,16 +1784,32 @@ export class HostReconciler {credentialRecord: k8s.V1Secret | null,options: BootstrapOptions): boolean {- if (options.forceFreshForWake || !deployment) return true+ if (options.forceFreshForWake || options.forceFreshForDeploymentChange || !deployment)+ return true// This reconcile will converge the Deployment to zero, so no Pod will// consume the stored material before the next verified wake.if (options.targetSuspended === true) return falseif ((deployment.spec?.replicas ?? 1) === 0) return true+ const annotations = credentialRecord?.metadata?.annotations ?? {}+ const fresh =+ annotations[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] === RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH+ const boundUid = annotations[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]+ if (boundUid && boundUid !== deployment.metadata?.uid) return trueif (HostReconciler.deploymentReady(deployment)) return falseif (- credentialRecord?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===- RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH &&- HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment)+ fresh &&+ credentialRecord &&+ HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) &&+ annotations[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] ===+ HostReconciler.deploymentRuntimeTokenRevision(deployment)+ )+ return true+ if (+ fresh &&+ credentialRecord &&+ (HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) ||+ (boundUid === deployment.metadata?.uid &&+ annotations[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true'))) {return false}@@ -1960,22 +2061,22 @@ export class HostReconciler {)return null}- // A fresh bootstrap is consumed here when it is bound to this- // Deployment (a wake bootstrap) or when a Ready Deployment already- // runs its revision (a normal mint whose pod booted before channel- // authority was lost). The pod rotates the single-use refresh token- // it booted with, so leaving the record fresh would later roll a new- // pod onto that revoked token. The pass returns after the one write;- // GFS renewal follows on the next hold pass.- if (- existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===- RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH &&- (HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment) ||- (HostReconciler.deploymentReady(deployment) &&- HostReconciler.deploymentRuntimeTokenRevision(deployment) === existingRevision))- ) {- const appliedRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)- if (HostReconciler.deploymentBootstrapConverged(deployment)) {+ const annotationUpdates = HostReconciler.bootstrapObservationUpdates(+ host,+ existing,+ deployment,+ existingRevision ?? ''+ )+ const renewalDue = HostReconciler.credentialRenewalWindowReached(+ existing,+ nowMs,+ GFS_TOKEN_REFRESH_BEFORE_ANNOTATION+ )+ // A pending strict acknowledgement cannot starve independent GFS+ // renewal. Bind the applied bootstrap before changing the full+ // content hash; runtime bytes and the applied revision stay intact.+ if (!renewalDue || !HostReconciler.deploymentReady(deployment)) {+ if (Object.keys(annotationUpdates).length > 0) {await this.coreApi.replaceNamespacedSecret({name,namespace: host.namespace,@@ -1985,28 +2086,17 @@ export class HostReconciler {...existing.metadata,annotations: {...(existing.metadata?.annotations ?? {}),- [RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]:- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED,- [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',+ ...annotationUpdates,},},},})}- return {- revision: appliedRevision || existingRevision || '',- scopeHash:- existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',- }- }- if (!HostReconciler.deploymentReady(deployment)) return null- if (- !HostReconciler.credentialRenewalWindowReached(- existing,- nowMs,- GFS_TOKEN_REFRESH_BEFORE_ANNOTATION+ if (+ !HostReconciler.deploymentReady(deployment) &&+ !HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment))- ) {+ return nullthis.recordGfsLifecycleEvidence(host, 'reused')return {revision:@@ -2034,6 +2124,7 @@ export class HostReconciler {...existing.metadata,annotations: {...(existing.metadata?.annotations ?? {}),+ ...annotationUpdates,[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),@@ -2125,31 +2216,6 @@ export class HostReconciler {}}- if (- existingRevision &&- !decided.refresh &&- options.prepareForResourceChange &&- HostReconciler.deploymentBelongsToHost(deployment, host) &&- HostReconciler.deploymentBootstrapConverged(deployment) &&- !HostReconciler.deploymentResourcesMatchHost(deployment!, host) &&- !(- bootstrapIsFresh &&- existing?.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] ===- 'true' &&- HostReconciler.deploymentRuntimeTokenRevision(deployment) !== existingRevision- )- ) {- // A resource edit replaces the Ready pod and loses its rotated- // in-memory lineage. Persist usable bootstrap before that replace.- // A valid fresh pending revision already prepares the same retry;- // scope, identity and expiry still pass the decision above.- decided = {- refresh: true,- rolloutRequired: true,- reason: 'fresh_mint_for_resource_change',- }- }-// Revoked-on-wake guard: the mcp-host runtime refresh token is// single-use-rotating (control-api revokes the prior JTI on every// refresh). A pod that is about to boot -- wake (scale 0->1), missing@@ -2251,26 +2317,76 @@ export class HostReconciler {projection,grokProjection)- const decision = decideRefresh(observedFrontsOAuth)+ let decision = decideRefresh(observedFrontsOAuth)+ const resourcesWillChange =+ options.prepareForResourceChange === true &&+ HostReconciler.deploymentBelongsToHost(deployment, host) &&+ (deployment.spec?.replicas ?? 1) > 0 &&+ !HostReconciler.deploymentResourcesMatchHost(deployment!, host)+ const bootstrapApplied =+ !!existing &&+ !!existingRevision &&+ HostReconciler.bootstrapAppliedToDeployment(existing, deployment, existingRevision)+ const boundUid =+ existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]+ const usablePendingPreparation =+ bootstrapIsFresh &&+ !bootstrapApplied &&+ (!boundUid || boundUid === deployment?.metadata?.uid)+ if (resourcesWillChange) {+ // Resource preparation is optional until scope authority is observed;+ // it must not participate in the intrinsic renewal/OAuth fallback above.+ // Compose a real transition with an already due renewal in one issuance.+ decision = {+ ...decision,+ refresh: decision.refresh || !usablePendingPreparation,+ rolloutRequired: true,+ reason: decision.refresh ? decision.reason : 'fresh_mint_for_resource_change',+ }+ }if (existing && existingRevision && !decision.refresh) {const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)const rolloutMarker =existing.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]+ const boundTargetRevision =+ existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]+ const preparedRevision =+ bootstrapIsFresh && boundUid === deployment?.metadata?.uid && boundTargetRevision+ ? boundTargetRevision+ : existingRevisionconst bootstrapBoundToAppliedDeployment =HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)- const bootstrapPendingForAppliedDeployment =- bootstrapIsFresh && bootstrapBoundToAppliedDeployment+ const bootstrapPendingForAppliedDeployment = bootstrapIsFresh && bootstrapAppliedconst bootstrapConsumedForAppliedDeployment =existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED && bootstrapBoundToAppliedDeploymentconst rolloutPending =- rolloutMarker === 'true' ||- (rolloutMarker === undefined &&- bootstrapIsFresh &&- deploymentRevision !== '' &&- deploymentRevision !== existingRevision)- const annotationUpdates: Record<string, string> = {}+ !bootstrapApplied &&+ (resourcesWillChange ||+ rolloutMarker === 'true' ||+ (rolloutMarker === undefined &&+ bootstrapIsFresh &&+ deploymentRevision !== '' &&+ deploymentRevision !== existingRevision))+ const annotationUpdates = HostReconciler.bootstrapObservationUpdates(+ host,+ existing,+ deployment,+ existingRevision+ )+ if (resourcesWillChange && usablePendingPreparation && !bootstrapApplied) {+ // Older preparations can already carry the rollout marker without+ // a target binding. Persist that binding before delivering them.+ const preparationAnnotations = {+ [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'true',+ [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preparedRevision,+ }+ for (const [key, value] of Object.entries(preparationAnnotations)) {+ if (existing.metadata?.annotations?.[key] !== value) annotationUpdates[key] = value+ }+ }if (cacheSynced) {const currentIngress =existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]@@ -2291,23 +2407,6 @@ export class HostReconciler {) {annotationUpdates[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)}- if (- bootstrapPendingForAppliedDeployment &&- HostReconciler.deploymentBootstrapConverged(deployment)- ) {- annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] =- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED- annotationUpdates[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] = 'false'- }- if (- bootstrapIsFresh &&- HostReconciler.deploymentBootstrapConverged(deployment) &&- deploymentRevision === existingRevision- ) {- annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] =- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED- annotationUpdates[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] = 'false'- }if (Object.keys(annotationUpdates).length > 0) {await this.coreApi.replaceNamespacedSecret({name,@@ -2340,15 +2439,24 @@ export class HostReconciler {(!bootstrapPendingForAppliedDeployment &&!bootstrapConsumedForAppliedDeployment &&HostReconciler.shouldRollForRuntimeSecret(deployment, false))- ? existingRevision+ ? preparedRevision: deploymentRevision || existingRevisionreturn {revision: selectedRevision,scopeHash,+ deploymentUid: deployment?.metadata?.uid,+ freshBootstrapPrepared:+ bootstrapIsFresh &&+ !bootstrapApplied &&+ (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION] ?? boundUid) ===+ deployment?.metadata?.uid &&+ (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION] ??+ boundTargetRevision) === selectedRevision,}}if (+ !resourcesWillChange &&existing &&existingRevision &&(decision.reason === 'refresh_before_reached' ||@@ -2369,6 +2477,7 @@ export class HostReconciler {return {revision: HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision,scopeHash,+ deploymentUid: deployment?.metadata?.uid,}}@@ -2411,12 +2520,14 @@ export class HostReconciler {),[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',- ...(preserveWakeTemplate && deployment+ ...((preserveWakeTemplate || rolloutRequired) &&+ HostReconciler.deploymentBelongsToHost(deployment, host)? {[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]:deployment.metadata?.uid ?? '',- [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]:- HostReconciler.deploymentRuntimeTokenRevision(deployment),+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preserveWakeTemplate+ ? HostReconciler.deploymentRuntimeTokenRevision(deployment)+ : revision,}: {}),[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),@@ -2440,6 +2551,9 @@ export class HostReconciler {? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision: revision,scopeHash,+ deploymentUid: deployment?.metadata?.uid,+ freshBootstrapPrepared:+ rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),}}@@ -2470,6 +2584,9 @@ export class HostReconciler {? revision: HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision,scopeHash,+ deploymentUid: deployment?.metadata?.uid,+ freshBootstrapPrepared:+ rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),}} catch (err) {lastErr = err@@ -4082,16 +4199,27 @@ export class HostReconciler {mounts: ResolvedSfsMount[],runtimeTokenRevision: string | undefined,lifecycle?: EffectiveHostLifecycle,- resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>,+ resolveStateBeforeMutation?: (observed?: k8s.V1Deployment) => Promise<DeploymentMutationState>,revalidate?: () => void): Promise<boolean> {let observedDeployment: k8s.V1Deployment | undefinedlet holdingTemplate = falselet applied = trueconst buildDesiredDeployment = async (): Promise<k8s.V1Deployment | null> => {- const state = resolveStateBeforeMutation ? await resolveStateBeforeMutation() : null+ const state = resolveStateBeforeMutation+ ? await resolveStateBeforeMutation(observedDeployment)+ : nullconst effective = state?.lifecycle ?? lifecycle- holdingTemplate = effective?.suspensionBlocked === true+ const resourcesDiffer =+ !!observedDeployment &&+ HostReconciler.deploymentBelongsToHost(observedDeployment, host) &&+ !HostReconciler.deploymentResourcesMatchHost(observedDeployment, host)+ const suspendingWithResourceDrift =+ effective?.stateless === true &&+ effective.state === 'suspended' &&+ effective.suspensionBlocked !== true &&+ resourcesDiffer+ holdingTemplate = effective?.suspensionBlocked === true || suspendingWithResourceDriftif (holdingTemplate) {if (!observedDeployment) return nullconst existing = observedDeployment@@ -4111,22 +4239,50 @@ export class HostReconciler {if (!hostUidVerified) {throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)}- // Preserve UID/resourceVersion and every applied field. Conflicts- // re-read this object before retrying; replicas is the only field that- // may change. An unverified legacy object must never be scaled.- const replicas = effective?.allowScaleUpDuringHold ? 1 : (existing.spec.replicas ?? 1)+ // Preserve UID/resourceVersion and every applied field. A real suspend+ // must still reach zero when resource policy changes; its next wake+ // prepares credentials before applying that deferred template change.+ // Conflicts re-read identity before retrying the replica-only mutation.+ const replicas = suspendingWithResourceDrift+ ? 0+ : effective?.allowScaleUpDuringHold+ ? 1+ : (existing.spec.replicas ?? 1)return {...existing,spec: { ...existing.spec, replicas },}}- return this.buildDeployment(+ if (+ observedDeployment &&+ resourcesDiffer &&+ (observedDeployment.spec?.replicas ?? 1) > 0 &&+ effective?.state === 'active' &&+ state?.freshBootstrapPrepared !== true+ ) {+ // Readiness cannot gate a resource repair that is needed for scheduling.+ // Delivery is gated on persisted, fresh material for the observed target.+ throw new RuntimeBootstrapPreparationUnavailableError()+ }+ const desired = this.buildDeployment(host,mounts,state?.runtimeTokenRevision ?? runtimeTokenRevision,effective,state?.grokExecutionEnabled ?? this.hostDerivesGrokExecution(host))+ if (observedDeployment && resourcesDiffer && effective?.state === 'draining') {+ // Defer only resource policy while draining. An intrinsic credential+ // recovery must still deliver its prepared runtime revision.+ const appliedResources = observedDeployment.spec!.template.spec!.containers.find(+ container => container.name === 'mcp-host'+ )?.resources+ const desiredContainer = desired.spec!.template.spec!.containers.find(+ container => container.name === 'mcp-host'+ )!+ desiredContainer.resources = structuredClone(appliedResources)+ }+ return desired}const mutationAllowed = () => {revalidate?.()@@ -5575,7 +5731,9 @@ export class HostReconciler {)}- const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {+ const resolveDeploymentState = async (+ observed?: k8s.V1Deployment+ ): Promise<DeploymentMutationState> => {for (let attempt = 1; attempt <= 3; attempt++) {revalidateHostMutationBoundary()const beforeScope = await resolveDeploymentLifecycle()@@ -5587,6 +5745,28 @@ export class HostReconciler {}}await ensureCurrentRuntimeTokenScope()+ const deploymentIdentityChanged =+ !!observed?.metadata?.uid &&+ !!requireRuntimeTokenProvision().deploymentUid &&+ observed.metadata.uid !== requireRuntimeTokenProvision().deploymentUid+ const observedResourceTransition =+ !!observed &&+ beforeScope.state === 'active' &&+ HostReconciler.deploymentBelongsToHost(observed, host) &&+ (observed.spec?.replicas ?? 1) > 0 &&+ !HostReconciler.deploymentResourcesMatchHost(observed, host)+ if (deploymentIdentityChanged || observedResourceTransition) {+ // A CAS retry or existence read can reveal a new UID or resource drift.+ // Prepare against this observation before rebuilding the Pod template;+ // persisted preparation is reused for retries against the same target.+ runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {+ forceFreshForDeploymentChange: deploymentIdentityChanged,+ targetSuspended: beforeScope.stateless && beforeScope.state === 'suspended',+ prepareForResourceChange: beforeScope.state === 'active',+ })+ requireRuntimeTokenProvision()+ revalidateHostMutationBoundary()+ }const effective = await resolveDeploymentLifecycle()const currentOAuth = await this.observeFrontsOAuthServer(host)// The OAuth lookup can await I/O after the last lifecycle observation.@@ -5602,6 +5782,7 @@ export class HostReconciler {return {lifecycle: effective,runtimeTokenRevision: requireRuntimeTokenProvision().revision,+ freshBootstrapPrepared: requireRuntimeTokenProvision().freshBootstrapPrepared,grokExecutionEnabled: this.hostDerivesGrokExecution(host),}}@@ -5628,12 +5809,19 @@ export class HostReconciler {revalidateHostMutationBoundary)} catch (error) {- if (error instanceof Error && error.name === 'RuntimeScopeObservationUnavailableError') {+ if (+ error instanceof Error &&+ (error.name === 'RuntimeScopeObservationUnavailableError' ||+ error.name === 'RuntimeBootstrapPreparationUnavailableError')+ ) {const liveDeployment = await this.readHostDeploymentOrNull(host)this.setStatus(host.name, {deployed: !!liveDeployment,ready: false,- message: 'Waiting for authoritative scope observation',+ message:+ error.name === 'RuntimeBootstrapPreparationUnavailableError'+ ? error.message+ : 'Waiting for authoritative scope observation',})return}diff --git a/host-context-controller/src/kubernetesQuantity.test.ts b/host-context-controller/src/kubernetesQuantity.test.tsnew file mode 100644index 000000000..d7d6e4bc8--- /dev/null+++ b/host-context-controller/src/kubernetesQuantity.test.ts@@ -0,0 +1,156 @@+import { describe, expect, it } from 'vitest'+import { kubernetesQuantitiesEqual } from './kubernetesQuantity'++describe('kubernetesQuantitiesEqual', () => {+ it.each([+ ['1000m', '1'],+ ['0.025', '25m'],+ ['1000n', '1u'],+ ['1000u', '1m'],+ ['1k', '1000'],+ ['1M', '1000k'],+ ['1G', '1000M'],+ ['1T', '1000G'],+ ['1P', '1000T'],+ ['1E', '1000P'],+ ['1Ki', '1024'],+ ['1024Ki', '1Mi'],+ ['1024Mi', '1Gi'],+ ['1024Gi', '1Ti'],+ ['1024Ti', '1Pi'],+ ['1024Pi', '1Ei'],+ ['1.5Gi', '1536Mi'],+ ['100.035Ki', '102435.84'],+ ['0.025Ti', '27487790694.4'],+ ['1e3', '1k'],+ ['1E-3', '1m'],+ ['1.3E+6', '1300k'],+ ['025.000e-3', '+25m'],+ ['.001', '1m'],+ ['1.', '1'],+ ['1.G', '1G'],+ ['-1000m', '-1'],+ ['-1Gi', '-1024Mi'],+ ['-0', '+0.000Gi'],+ ['0e2147483647', '0n'],+ ])('recognizes %s and %s as equal', (left, right) => {+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)+ expect(kubernetesQuantitiesEqual(right, left)).toBe(true)+ })++ it.each([+ ['1', '1001m'],+ ['1k', '1Ki'],+ ['1M', '1Mi'],+ ['25m', '0.026'],+ ['1', '-1'],+ ['0', '1n'],+ ['9007199254740992', '9007199254740993'],+ ['1.000000001', '1'],+ ['1Gi', '1073741824.000000001'],+ ['12E', '9223372036854775807'],+ ])('distinguishes %s and %s', (left, right) => {+ expect(kubernetesQuantitiesEqual(left, right)).toBe(false)+ expect(kubernetesQuantitiesEqual(right, left)).toBe(false)+ })++ it.each([+ ['3.001n', '4n'],+ ['1.1E-9', '2n'],+ ['0.0000000001', '1n'],+ ['0.00000012345', '124n'],+ ['0.000000000001Ki', '2n'],+ ['-0.0000000001', '-1n'],+ ['-3.001n', '-4n'],+ ['1.0000000001', '1.000000001'],+ ['9.9999999991', '10'],+ ['-9.9999999991', '-10'],+ ['0.0000000010000', '1n'],+ ['0.0000000000000', '0'],+ ])('rounds %s away from zero at nano precision to %s', (left, right) => {+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)+ })++ it.each([+ ['9Ei', '9223372036854775807'],+ ['9223372036854775807Ki', '9223372036854775807'],+ ['-9Ei', '-9223372036854775807'],+ ['7.9999999999999999999Ei', '9223372036854775807'],+ ['12E', '12000000000000000000'],+ ['9223372036854775808', '9223372036854775808e0'],+ ])('applies format-specific maximum rules to %s and %s', (left, right) => {+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)+ })++ it.each([+ '',+ '+',+ '-',+ '.',+ '-.',+ 'm',+ 'Ki',+ '+Gi',+ '1.1.M',+ '1+1.0M',+ '0.1mi',+ '0.1am',+ 'aoeu',+ '.5i',+ '1i',+ '-3.01e-',+ '1e',+ '1K',+ '1e1.5',+ ' 1',+ '1 ',+ '1\n',+ 'NaN',+ 'Infinity',+ '1e9223372036854775808',+ '1e-9223372036854775809',+ ])('rejects invalid quantities with a fixed error (%j)', value => {+ expect(() => kubernetesQuantitiesEqual(value, '1')).toThrowError(+ new Error('Invalid Kubernetes quantity')+ )+ expect(() => kubernetesQuantitiesEqual('1', value)).toThrowError(+ new Error('Invalid Kubernetes quantity')+ )+ expect(() => kubernetesQuantitiesEqual(value, value)).toThrowError(+ new Error('Invalid Kubernetes quantity')+ )+ })++ it('compares large positive exponents without expanding powers of ten', () => {+ expect(kubernetesQuantitiesEqual('1e1000000000', '10e999999999')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e1000000000', '1e999999999')).toBe(false)+ expect(kubernetesQuantitiesEqual('1e2147483647', '10e2147483646')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e-1000000000', '1n')).toBe(true)+ expect(kubernetesQuantitiesEqual('-1e-1000000000', '-1n')).toBe(true)+ })++ it('preserves the upstream int64-to-int32 exponent narrowing', () => {+ expect(kubernetesQuantitiesEqual('1e4294967296', '1')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e4294967295', '0.1')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e9223372036854775807', '0.1')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e-9223372036854775808', '1')).toBe(true)+ expect(kubernetesQuantitiesEqual('1.0e-2147483648', '10e2147483647')).toBe(true)+ })++ it('preserves the slow parser nano scale wrapping at int32 boundaries', () => {+ expect(kubernetesQuantitiesEqual('1000000000000000000e2147483647', '1n')).toBe(true)+ expect(kubernetesQuantitiesEqual('1e-2147483648', '1n')).toBe(true)+ })++ it('rejects exponents outside int64 before creating an exponent BigInt', () => {+ const value = `1e${'9'.repeat(10000)}`+ expect(() => kubernetesQuantitiesEqual(value, '1')).toThrowError(+ new Error('Invalid Kubernetes quantity')+ )+ })++ it('handles long zero padding without expanding the exponent', () => {+ expect(kubernetesQuantitiesEqual(`1e+${'0'.repeat(10000)}1`, '10')).toBe(true)+ expect(kubernetesQuantitiesEqual(`1${'0'.repeat(10000)}e-10000`, '1')).toBe(true)+ })+})diff --git a/host-context-controller/src/kubernetesQuantity.ts b/host-context-controller/src/kubernetesQuantity.tsnew file mode 100644index 000000000..78d8ac8dc--- /dev/null+++ b/host-context-controller/src/kubernetesQuantity.ts@@ -0,0 +1,89 @@+const decimalSuffixes = ['n', 'u', 'm', '', 'k', 'M', 'G', 'T', 'P', 'E']+const binarySuffixes = ['Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei']+const maximumBinaryQuantity = '9223372036854775807'++function invalidQuantity(): never {+ throw new Error('Invalid Kubernetes quantity')+}++function decimalExponent(suffix: string): number {+ const negative = suffix[1] === '-'+ const digits = suffix.slice(1).replace(/^[+-]/, '').replace(/^0+/, '') || '0'+ const maximum = negative ? '9223372036854775808' : '9223372036854775807'+ if (digits.length > maximum.length || (digits.length === maximum.length && digits > maximum)) {+ invalidQuantity()+ }+ // Apimachinery parses a signed int64 exponent, then narrows it to int32.+ return Number(BigInt.asIntN(32, BigInt(`${negative ? '-' : ''}${digits}`)))+}++function normalizedQuantity(value: string): string {+ const match =+ /^([+-]?)([0-9]*)(?:\.([0-9]*))?(n|u|m|k|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei|[eE][+-]?[0-9]+)?$/.exec(+ value+ )+ // Require a digit as the Quantity grammar does, rather than accepting the+ // Go scanner's undocumented bare-sign, bare-dot, or bare-suffix zero aliases.+ if (!match || match[0] !== value || !(match[2] || match[3])) {+ invalidQuantity()+ }++ const numerator = match[2].replace(/^0+/, '') || '0'+ const fraction = match[3] || ''+ const suffix = match[4] || ''+ const binaryIndex = binarySuffixes.indexOf(suffix)+ const decimalIndex = decimalSuffixes.indexOf(suffix)+ let digits = `${numerator}${fraction}`.replace(/^0+/, '') || '0'+ let exponent = -fraction.length++ if (binaryIndex >= 0) {+ // BinarySI has only six fixed exponents, so this shift cannot grow with an+ // exponent supplied by the caller.+ digits = (BigInt(digits) << BigInt((binaryIndex + 1) * 10)).toString()+ } else {+ const suffixExponent = decimalIndex >= 0 ? (decimalIndex - 3) * 3 : decimalExponent(suffix)+ exponent = (suffixExponent - fraction.length) | 0+ if (numerator.length + fraction.length > 18 || exponent < -9) {+ // The slow decimal parser rounds to nano using int32 scale arithmetic.+ // Preserve its scale wrapping without constructing its power of ten.+ exponent = ((exponent + 9) | 0) - 9+ }+ }++ if (digits === '0') return '0'++ if (exponent < -9) {+ const keptDigits = digits.length + exponent + 9+ if (keptDigits <= 0) {+ digits = '1'+ } else {+ const remainder = /[1-9]/.test(digits.slice(keptDigits))+ digits = (BigInt(digits.slice(0, keptDigits)) + (remainder ? 1n : 0n)).toString()+ }+ exponent = -9+ }++ const withoutTrailingZeroes = digits.replace(/0+$/, '')+ exponent += digits.length - withoutTrailingZeroes.length+ digits = withoutTrailingZeroes++ // ParseQuantity caps only BinarySI; DecimalSI and exponent quantities can+ // exceed int64. After nano rounding, this comparison needs at most 28 digits.+ if (+ binaryIndex >= 0 &&+ (digits.length + exponent > maximumBinaryQuantity.length ||+ (digits.length + exponent === maximumBinaryQuantity.length &&+ digits.padEnd(maximumBinaryQuantity.length, '0') > maximumBinaryQuantity))+ ) {+ digits = maximumBinaryQuantity+ exponent = 0+ }++ // Keep powers of ten as an integer exponent, including very large values.+ return `${match[1] === '-' ? '-' : ''}${digits}e${exponent}`+}++/** Compare numeric Quantity values using apimachinery v0.34.0 precision rules. */+export function kubernetesQuantitiesEqual(left: string, right: string): boolean {+ return normalizedQuantity(left) === normalizedQuantity(right)+}diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 43e2a1d72..d1cb138bd 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1,5 +1,6 @@import { afterEach, describe, expect, it, vi } from 'vitest'import * as k8s from '@kubernetes/client-node'+import { isDeepStrictEqual } from 'node:util'import { asApiserverDeployment } from '../src/__tests__/asApiserverDeployment'import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeReporter'import { config } from '../src/config'@@ -534,8 +535,11 @@ function rejectedCondition(status: HostCrdStatus) {describe('HostReconciler resource rollout bootstrap', () => {const initialResources = structuredClone(config.hostResources)const initialDesktopResources = structuredClone(config.desktopResources)+ const initialRefreshBefore = config.mcpHostBootstrapRefreshBeforeSecconst issue = vi.mocked(issueMcpHostRuntimeTokens)const initialIssue = issue.getMockImplementation()!+ const gfsMint = vi.mocked(mintHostGfsToken)+ const initialGfsMint = gfsMint.getMockImplementation()!const BOOTSTRAP = 'clerum.io/runtime-token-bootstrap-state'const SECRET_REVISION = 'clerum.io/runtime-token-secret-revision'const APPLIED_REVISION = 'clerum.io/runtime-token-revision'@@ -545,7 +549,9 @@ describe('HostReconciler resource rollout bootstrap', () => {afterEach(() => {config.hostResources = structuredClone(initialResources)config.desktopResources = structuredClone(initialDesktopResources)+ config.mcpHostBootstrapRefreshBeforeSec = initialRefreshBeforeissue.mockImplementation(initialIssue)+ gfsMint.mockImplementation(initialGfsMint)vi.useRealTimers()// Full reconciles schedule readiness polls; keep their timers inside this fixture.for (const reconciler of fixtures.splice(0)) {@@ -554,15 +560,29 @@ describe('HostReconciler resource rollout bootstrap', () => {})async function runtime(- options: { desktop?: boolean; stateless?: boolean; consumed?: boolean } = {}+ options: {+ desktop?: boolean+ stateless?: boolean+ draining?: boolean+ consumed?: boolean+ oauth?: boolean+ canonicalResources?: boolean+ } = {}) {vi.useFakeTimers({ toFake: ['Date'] })vi.setSystemTime(new Date('2026-07-03T00:00:00.000Z'))+ // A finite producer-derived renewal window exercises maintenance before expiry.+ config.mcpHostBootstrapRefreshBeforeSec = 90const host = makeHost({generation: 1,...(options.stateless? makeStatelessHost({- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },+ status: {+ lifecycle: {+ state: options.draining ? 'draining' : 'active',+ wakeHandledGeneration: 0,+ },+ },}): {}),...(options.desktop ? { spec: { ...makeHost().spec, desktop: { browser: true } } } : {}),@@ -571,7 +591,7 @@ describe('HostReconciler resource rollout bootstrap', () => {const apis = createReconciler({ isCommunicationChannelCacheSynced: () => cacheSynced })const { reconciler, appsApi, coreApi, customApi } = apisfixtures.push(reconciler)- reconciler.setHostFrontsOAuthServer(async () => false)+ reconciler.setHostFrontsOAuthServer(async () => options.oauth ?? false)const readHost = customApi.getNamespacedCustomObject.getMockImplementation()!customApi.getNamespacedCustomObject.mockImplementation(async request =>request.plural === 'hosts'@@ -583,6 +603,7 @@ describe('HostReconciler resource rollout bootstrap', () => {)let issuance = 0+ let gfsIssuance = 0const events: string[] = []// Only the issuer boundary is simulated. Distinct synthetic emissions and// decodable future refresh expiries exercise the real persistence/reuse path.@@ -596,10 +617,17 @@ describe('HostReconciler resource rollout bootstrap', () => {mcpHostControlToken: `synthetic-control-${emission}`,}})+ // Independent synthetic GFS emissions expose content-hash changes without+ // changing the runtime access/refresh/control material.+ gfsMint.mockImplementation(async (...args) => ({+ ...(await initialGfsMint(...args)),+ token: `synthetic-gfs-${++gfsIssuance}`,+ }))let secret: k8s.V1Secret | null = nulllet deployment: k8s.V1Deployment | null = nulllet secretVersion = 0let deploymentVersion = 0+ let templateChanges = 0let replacementFailures: number[] = []let secretConflict = falseconst readSecret = coreApi.readNamespacedSecret.getMockImplementation()!@@ -653,6 +681,31 @@ describe('HostReconciler resource rollout bootstrap', () => {})const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ const persistDeployment = (body: k8s.V1Deployment) => {+ const result = asApiserverDeployment(body)+ // The recorded-default overlay does not merge Date objects. A real SDK+ // read preserves these timestamps on a replica-only replacement.+ for (const [incoming, persisted] of [+ [body.metadata, result.metadata],+ [body.spec?.template.metadata, result.spec?.template.metadata],+ ]) {+ if (incoming?.creationTimestamp instanceof Date && persisted) {+ persisted.creationTimestamp = structuredClone(incoming.creationTimestamp)+ }+ }+ if (options.canonicalResources) {+ // Closed equivalences from apimachinery Quantity parse/serialization,+ // independent of the production guard's implementation.+ for (const container of result.spec!.template.spec!.containers) {+ for (const kind of ['requests', 'limits'] as const) {+ const values = container.resources?.[kind]+ if (values?.cpu === '1000m') values.cpu = '1'+ if (values?.memory === '1024Mi') values.memory = '1Gi'+ }+ }+ }+ return result+ }appsApi.readNamespacedDeployment.mockImplementation(async request => {if (request.name !== host.name) return readDeployment(request)if (!deployment) throw { code: 404 }@@ -663,7 +716,7 @@ describe('HostReconciler resource rollout bootstrap', () => {const { body } = requestif (body.metadata?.name !== host.name) return createDeployment(request)if (deployment) throw { code: 409 }- deployment = asApiserverDeployment(body)+ deployment = persistDeployment(body)deployment.metadata = {...deployment.metadata,uid: 'runtime-deployment-uid',@@ -686,13 +739,16 @@ describe('HostReconciler resource rollout bootstrap', () => {throw { code: failure }}const previous = deployment- deployment = asApiserverDeployment(body)+ deployment = persistDeployment(body)deployment.metadata = {...deployment.metadata,uid: previous.metadata!.uid,resourceVersion: String(++deploymentVersion),- generation: previous.metadata!.generation! + 1,+ generation:+ previous.metadata!.generation! ++ (isDeepStrictEqual(previous.spec, deployment.spec) ? 0 : 1),}+ if (!isDeepStrictEqual(previous.spec!.template, deployment.spec!.template)) templateChanges++// Kubernetes status belongs to the prior generation until advance() is called.deployment.status = structuredClone(previous.status)events.push('replace-deployment')@@ -719,6 +775,7 @@ describe('HostReconciler resource rollout bootstrap', () => {expect(secret!.metadata!.annotations![BOOTSTRAP]).toBe('consumed')}events.length = 0+ templateChanges = 0issue.mockClear()return {...apis,@@ -734,9 +791,20 @@ describe('HostReconciler resource rollout bootstrap', () => {conflictSecret: () => {secretConflict = true},+ unbindLegacyPreparation: () => {+ delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']+ delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-applied-revision']+ secret!.metadata!.resourceVersion = String(++secretVersion)+ },setCacheSynced: (value: boolean) => {cacheSynced = value},+ templateChanges: () => templateChanges,+ recreateDeployment: () => {+ deployment!.metadata!.uid = 'recreated-runtime-deployment-uid'+ deployment!.metadata!.resourceVersion = String(++deploymentVersion)+ deployment!.metadata!.generation = 1+ },}}@@ -907,6 +975,361 @@ describe('HostReconciler resource rollout bootstrap', () => {await fixture.reconcile()expect(issue).toHaveBeenCalledOnce()})++ it.each(['Desktop CPU', 'Host memory'])(+ 'keeps equivalent canonical API quantities stable: %s',+ async kind => {+ const desktop = kind === 'Desktop CPU'+ if (!desktop) config.hostResources.requests.memory = '1024Mi'+ const fixture = await runtime({ desktop, canonicalResources: true, consumed: false })+ for (let pass = 0; pass < 3; pass++) {+ await fixture.reconcile()+ fixture.advance()+ }+ expect(issue).not.toHaveBeenCalled()+ expect(fixture.templateChanges()).toBe(0)+ expect(fixture.deployment().metadata!.generation).toBe(1)+ }+ )++ it('defers resource preparation when retained OAuth authority is temporarily unobserved', async () => {+ const fixture = await runtime({ oauth: true })+ config.hostResources.requests.cpu = '25m'+ fixture.reconciler.setHostFrontsOAuthServer(async () => {+ throw new Error('simulated OAuth observation unavailable')+ })+ await fixture.reconcile()+ expect(issue).not.toHaveBeenCalled()+ expect(fixture.events).toEqual([])+ expect(+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-fronts-oauth-server']+ ).toBe('true')+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(+ '100m'+ )+ fixture.reconciler.setHostFrontsOAuthServer(async () => true)+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(issue.mock.calls[0][2]).toContain(OAUTH_USER_TOKEN_SCOPE)+ expect(fixture.templateChanges()).toBe(1)+ })++ it('reuses an intrinsic renewal persisted before a failed resource replacement', async () => {+ const fixture = await runtime()+ vi.setSystemTime(+ Date.parse(+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-refresh-before']+ ) + 1+ )+ config.hostResources.requests.cpu = '25m'+ fixture.failReplacements(503)+ await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })+ const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ prepared+ )+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('binds and delivers legacy unbound preparation even when its rollout marker is already true', async () => {+ const fixture = await runtime()+ config.hostResources.requests.cpu = '25m'+ fixture.failReplacements(503)+ await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })+ // The previously shipped preparation did not bind its target until acknowledgement.+ fixture.unbindLegacyPreparation()+ fixture.events.length = 0+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.events).toEqual(['persist-secret', 'replace-deployment'])+ expect(+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']+ ).toBe(fixture.deployment().metadata!.uid)+ for (let pass = 0; pass < 3; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('applies and acknowledges the preparation when intrinsic renewal and resources coincide', async () => {+ const fixture = await runtime()+ vi.setSystemTime(+ Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-refresh-before']) + 1+ )+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ prepared+ )+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('renews held GFS while strict bootstrap acknowledgement remains pending', async () => {+ const fixture = await runtime({ stateless: true, consumed: false })+ const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]+ const runtimeData = { ...fixture.secret().data }+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })+ fixture.setCacheSynced(false)+ gfsMint.mockClear()+ vi.setSystemTime(Date.now() + 550_000)+ await fixture.reconcile()+ await fixture.reconcile()+ vi.setSystemTime(Date.now() + 100_000)+ await fixture.reconcile()+ expect(gfsMint).toHaveBeenCalledOnce()+ expect(issue).not.toHaveBeenCalled()+ expect(+ Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-expires-at'])+ ).toBeGreaterThan(Date.now())+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')+ for (const key of [+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,+ ]) {+ expect(fixture.secret().data![key]).toBe(runtimeData[key])+ }+ expect(fixture.secret().data![MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(+ runtimeData[MCP_HOST_GFS_TOKEN_SECRET_KEY]+ )+ fixture.setCacheSynced(true)+ await fixture.reconcile()+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ applied+ )+ expect(fixture.templateChanges()).toBe(0)+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('recovers once after an observed partial Ready bootstrap subsequently loses Ready', async () => {+ const fixture = await runtime({ consumed: false })+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')+ fixture.advance({ readyReplicas: 0, availableReplicas: 0, updatedReplicas: 0 })+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.templateChanges()).toBe(1)+ })++ it('does not mistake stale-generation Ready for possible bootstrap use during slow startup', async () => {+ const fixture = await runtime({ consumed: false })+ fixture.advance({ observedGeneration: 0, updatedReplicas: 0, readyReplicas: 1 })+ await fixture.reconcile()+ fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })+ for (let pass = 0; pass < 5; pass++) await fixture.reconcile()+ expect(issue).not.toHaveBeenCalled()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')+ expect(fixture.templateChanges()).toBe(0)+ })++ it('prepares for a changed Deployment UID observed after credential persistence', async () => {+ const fixture = await runtime()+ config.hostResources.requests.cpu = '25m'+ const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!+ let reads = 0+ fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {+ if (request.name === fixture.host.name && ++reads === 2) fixture.recreateDeployment()+ return read(request)+ })+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledTimes(2)+ expect(+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']+ ).toBe('recreated-runtime-deployment-uid')+ expect(fixture.templateChanges()).toBe(1)+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledTimes(2)+ })++ it('does not reuse a consumed record bound to a previous Deployment UID', async () => {+ const fixture = await runtime()+ fixture.recreateDeployment()+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']+ ).toBe('recreated-runtime-deployment-uid')+ await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ })++ it('replans resource preparation against drift observed at the mutation read with the same UID', async () => {+ const fixture = await runtime()+ const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!+ let reads = 0+ fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {+ if (request.name === fixture.host.name && ++reads === 2) {+ fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'+ fixture.deployment().metadata!.generation!+++ fixture.advance()+ }+ return read(request)+ })+ await fixture.reconcile()+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(+ '100m'+ )+ })++ it('prepares a consumed-bootstrap resource transition without acknowledging a partial Ready runtime', async () => {+ const fixture = await runtime()+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')+ for (let pass = 0; pass < 3; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.templateChanges()).toBe(1)+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('prepares an immediate resource repair for a fresh applied bootstrap without Ready replicas', async () => {+ const fixture = await runtime({ consumed: false })+ fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ fixture.secret().metadata!.annotations![SECRET_REVISION]+ )+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.templateChanges()).toBe(1)+ fixture.advance()+ await fixture.reconcile()+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')+ })++ it('recovers a NotReady consumed bootstrap when its resources also drift', async () => {+ const fixture = await runtime()+ fixture.advance({ readyReplicas: 0, availableReplicas: 0 })+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(+ '25m'+ )+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.templateChanges()).toBe(1)+ })++ it('replans same-UID resource drift introduced by a conflicting Deployment writer', async () => {+ config.desktopResources.requests.cpu = '1000m'+ const fixture = await runtime({ desktop: true, canonicalResources: true })+ const replace = fixture.appsApi.replaceNamespacedDeployment.getMockImplementation()!+ let changed = false+ fixture.appsApi.replaceNamespacedDeployment.mockImplementation(async request => {+ if (request.name === fixture.host.name && !changed) {+ changed = true+ fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'+ fixture.deployment().metadata!.generation!+++ fixture.advance()+ // The external writer's new resourceVersion makes this first PUT conflict.+ fixture.events.push('replace-409')+ }+ return replace(request)+ })+ await fixture.reconcile()+ expect(fixture.events).toEqual(['replace-409', 'issue', 'persist-secret', 'replace-deployment'])+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(+ '1'+ )+ expect(fixture.templateChanges()).toBe(1)+ })++ it('defers resource transitions while the owned stateless runtime is draining', async () => {+ const fixture = await runtime({ stateless: true, draining: true })+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(issue).not.toHaveBeenCalled()+ expect(fixture.events).toEqual([])+ expect(fixture.templateChanges()).toBe(0)+ expect(fixture.deployment().spec!.replicas).toBe(1)+ })++ it('delivers intrinsic bootstrap recovery while draining and preserves the applied resources', async () => {+ const fixture = await runtime({ stateless: true, draining: true })+ const resources = structuredClone(+ fixture.deployment().spec!.template.spec!.containers[0].resources+ )+ fixture.advance({ readyReplicas: 0, availableReplicas: 0 })+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ fixture.secret().metadata!.annotations![SECRET_REVISION]+ )+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(resources)+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()+ expect(issue).toHaveBeenCalledOnce()+ expect(fixture.templateChanges()).toBe(1)+ })++ it('suspends without applying resource drift or resurrecting the owned stateless runtime', async () => {+ const fixture = await runtime({ stateless: true })+ const template = structuredClone(fixture.deployment().spec!.template)+ const runtimeData = { ...fixture.secret().data }+ fixture.host.status = suspendedStatus()+ config.hostResources.requests.cpu = '25m'+ await fixture.reconcile()+ expect(fixture.deployment().spec!.replicas).toBe(0)+ expect(fixture.deployment().spec!.template).toEqual(template)+ expect(fixture.secret().data).toEqual(runtimeData)+ expect(issue).not.toHaveBeenCalled()+ expect(fixture.events).toEqual(['replace-deployment'])+ await fixture.reconcile()+ expect(fixture.deployment().spec!.replicas).toBe(0)+ expect(fixture.events).toEqual(['replace-deployment'])+ expect(issue).not.toHaveBeenCalled()+ })++ it('keeps consumed runtime credentials and the applied revision stable after held GFS renewal', async () => {+ const fixture = await runtime({ stateless: true })+ const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]+ const before = { ...fixture.secret().data }+ fixture.setCacheSynced(false)+ gfsMint.mockClear()+ vi.setSystemTime(Date.now() + 550_000)+ await fixture.reconcile()+ expect(gfsMint).toHaveBeenCalledOnce()+ expect(issue).not.toHaveBeenCalled()+ for (const key of [+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,+ ])+ expect(fixture.secret().data![key]).toBe(before[key])+ fixture.setCacheSynced(true)+ await fixture.reconcile()+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(+ applied+ )+ expect(fixture.templateChanges()).toBe(0)+ expect(issue).not.toHaveBeenCalled()+ })})describe('HostReconciler stateless lifecycle — buildDeployment replicas', () => {@@ -2203,12 +2626,18 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {it('does not let a failed live probe narrow a retained OAuth grant before its renewal window', async () => {const host = makeStatelessHost()- const { reconciler, coreApi } = createReconciler({+ const { reconciler, coreApi, appsApi } = createReconciler({isCommunicationChannelCacheSynced: () => true,})- coreApi.readNamespacedSecret.mockResolvedValue(- await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })- )+ const record = await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })+ coreApi.readNamespacedSecret.mockResolvedValue(record)+ const running = trustedRuntimeDeployment(reconciler, host)+ running.spec!.template.metadata!.annotations = {+ ...running.spec!.template.metadata!.annotations,+ 'clerum.io/runtime-token-revision':+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],+ }+ appsApi.readNamespacedDeployment.mockResolvedValue(running)const oauthResolver = vi.fn(async (): Promise<boolean> => {throw new Error('oauth lookup unavailable')})@@ -2870,7 +3299,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {return deployment}- it('marks a bootstrap bound to the applied Deployment consumed without minting', async () => {+ it('acknowledges a bound bootstrap and renews due GFS in one CAS write', async () => {const host = makeStatelessHost()const bound = await renewHeldGfs({@@ -2883,7 +3312,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(bound.recordReads).toBeGreaterThan(0)expect(bound.recordWrites).toBe(1)expect(bound.provision.revision).toBe(APPLIED_REVISION)- expect(bound.mints).toBe(0)+ expect(bound.mints).toBe(1)})it('renews once the bootstrap is bound to a different Deployment UID', async () => {@@ -2893,7 +3322,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {record: freshBootstrapBoundTo(`deployment-${host.name}`),deployment: appliedDeployment,})- expect(bound.mints).toBe(0)+ expect(bound.mints).toBe(1)expect(bound.recordWrites).toBe(1)const replaced = await renewHeldGfs({@@ -2970,10 +3399,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {vi.setSystemTime(t0 + 602_000)const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)- // Liveness witness: the hold renewed GFS once and wrote the record twice- // (the consumed marker, then the renewed GFS credential).+ // The independent renewal and strict acknowledgement share one CAS write.expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()- expect(coreApi.replaceNamespacedSecret.mock.calls.length).toBeGreaterThanOrEqual(2)+ expect(coreApi.replaceNamespacedSecret.mock.calls.length).toBe(1)expect(record.metadata!.annotations!['clerum.io/runtime-token-rollout-required']).not.toBe('true')