Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): separate bootstrap intent from runtime maintenance

Alfredo Lopez7dd1547ebf78
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index db02055f0..938e400a2 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.376",
+ "version": "0.1.377",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.376",
+ "version": "0.1.377",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index eb45e90ac..ef4a910fe 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.376",
+ "version": "0.1.377",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
index 1db6912e7..7afe1ba4a 100644
--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
@@ -184,7 +184,18 @@ function deploymentWithRevision(
replicas = 1
): k8s.V1Deployment {
return {
- metadata: { name: 'team-mission', namespace: 'mcp-host', generation: 1 },
+ metadata: {
+ name: 'team-mission',
+ namespace: 'mcp-host',
+ generation: 1,
+ uid: 'team-mission-deployment-uid',
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/host': 'team-mission',
+ },
+ annotations: { 'clerum.io/host-uid': 'team-mission-uid' },
+ },
spec: {
replicas,
template: {
@@ -371,7 +382,15 @@ describe('HostReconciler runtime credential Secret revision', () => {
expect(result.revision).toBe(currentSecretRevision)
expect(issueMcpHostRuntimeTokens).not.toHaveBeenCalled()
- expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
+ // Applied-target bookkeeping is one CAS write, with no credential change.
+ expect(coreApi.replaceNamespacedSecret).toHaveBeenCalledOnce()
+ const applied = replacedRuntimeSecret(coreApi)
+ expect(applied.metadata?.annotations?.[BOOTSTRAP_STATE_KEY]).toBe('fresh')
+ expect(applied.data).toEqual(secret.data)
+ Object.assign(secret, applied, {
+ metadata: { ...applied.metadata, resourceVersion: '20' },
+ })
+ coreApi.replaceNamespacedSecret.mockClear()
deployment.status!.readyReplicas = 1
await (
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index bc2597038..78fcce468 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -33,6 +33,7 @@ import {
WFC_APP_LABEL,
sharedFileSystemHash,
} from './k8s/sharedFileSystemFactory'
+import { kubernetesQuantitiesEqual } from './kubernetesQuantity'
import {
ALLOWED_MODELS_CONFIGMAP_NAME,
parseAllowedModelsSnapshot,
@@ -149,6 +150,13 @@ class HostMutationIdentityChangedError extends Error {
}
}
+class RuntimeBootstrapPreparationUnavailableError extends Error {
+ constructor() {
+ super('Waiting for a prepared runtime bootstrap before changing resources')
+ this.name = 'RuntimeBootstrapPreparationUnavailableError'
+ }
+}
+
class HostMutationSpecRevisionChangedError extends Error {
constructor(action: string, hostName: string) {
super(`Host spec generation changed before ${action} admission for "${hostName}"`)
@@ -396,6 +404,8 @@ const RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION =
'clerum.io/runtime-token-bootstrap-deployment-uid'
const RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION =
'clerum.io/runtime-token-bootstrap-applied-revision'
+const RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION =
+ 'clerum.io/runtime-token-bootstrap-ready-revision'
const RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION = 'clerum.io/runtime-token-has-channel-ingress'
const RUNTIME_TOKEN_FRONTS_OAUTH_ANNOTATION = 'clerum.io/runtime-token-fronts-oauth-server'
const RUNTIME_TOKEN_ISSUER = 'control-api'
@@ -503,6 +513,7 @@ type HostSecretValidationResult =
type BootstrapOptions = {
forceFreshForWake?: boolean
+ forceFreshForDeploymentChange?: boolean
targetSuspended?: boolean
prepareForResourceChange?: boolean
refreshGfsOnly?: boolean
@@ -512,6 +523,8 @@ type BootstrapOptions = {
type RuntimeTokenProvision = {
revision: string
scopeHash: string
+ deploymentUid?: string
+ freshBootstrapPrepared?: boolean
}
// Reported as the reconcile_outcome transition `gfs_token:<outcome>` (#328).
@@ -520,6 +533,7 @@ type GfsTokenLifecycleOutcome = 'minted' | 'rotated' | 'reused' | 'failed'
type DeploymentMutationState = {
lifecycle: EffectiveHostLifecycle
runtimeTokenRevision: string
+ freshBootstrapPrepared?: boolean
// Captured in the same synchronous step as the stable scope-hash check so
// the pod's Grok factory switch and the minted llm:grok:execute scope agree.
grokExecutionEnabled: boolean
@@ -1589,11 +1603,24 @@ export class HostReconciler {
host.spec.desktop?.browser || host.spec.desktop?.x11
? config.desktopResources
: config.hostResources
- // Match the Deployment comparator's exact quantity values, ignoring map order only.
- return isDeepStrictEqual(
- canonicalizeValue({ requests: resources?.requests, limits: resources?.limits }),
- canonicalizeValue(desired)
- )
+ let matches = true
+ for (const kind of ['requests', 'limits'] as const) {
+ const actual = resources?.[kind]
+ const expected = desired[kind]
+ if (!isDeepStrictEqual(Object.keys(actual ?? {}).sort(), Object.keys(expected).sort())) {
+ matches = false
+ }
+ for (const [key, value] of Object.entries(expected)) {
+ // Validate desired quantities even when a field is absent. A lexical
+ // PUT stays governed by the exact Deployment comparator, but equal
+ // Kubernetes amounts do not start a new Pod or need new credentials.
+ kubernetesQuantitiesEqual(value, value)
+ if (actual?.[key] === undefined || !kubernetesQuantitiesEqual(actual[key], value)) {
+ matches = false
+ }
+ }
+ }
+ return matches
}
/**
@@ -1626,6 +1653,64 @@ export class HostReconciler {
)
}
+ private static bootstrapAppliedToDeployment(
+ record: k8s.V1Secret,
+ deployment: k8s.V1Deployment | null,
+ contentRevision: string
+ ): boolean {
+ if (HostReconciler.bootstrapBindingMatchesDeployment(record, deployment)) return true
+ // An existing target binding must never be adopted by a same-name/new-UID
+ // Deployment. Older unbound records can be bound by their applied revision.
+ if (record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]) {
+ return false
+ }
+ return (
+ contentRevision !== '' &&
+ HostReconciler.deploymentRuntimeTokenRevision(deployment) === contentRevision
+ )
+ }
+
+ private static bootstrapObservationUpdates(
+ host: HostCRD,
+ record: k8s.V1Secret,
+ deployment: k8s.V1Deployment | null,
+ contentRevision: string
+ ): Record<string, string> {
+ if (
+ record.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] !==
+ RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH ||
+ !HostReconciler.deploymentBelongsToHost(deployment, host) ||
+ !HostReconciler.bootstrapAppliedToDeployment(record, deployment, contentRevision)
+ )
+ return {}
+ const current = record.metadata?.annotations ?? {}
+ const revision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
+ const desired: Record<string, string> = {
+ [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: revision,
+ // Application is distinct from consumption: GFS content may change
+ // while the same bootstrap waits for strict acknowledgement.
+ [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',
+ }
+ if (HostReconciler.deploymentBootstrapConverged(deployment)) {
+ desired[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] = RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED
+ } else if (
+ (deployment!.spec?.replicas ?? 1) > 0 &&
+ (deployment!.metadata!.generation ?? 0) > 0 &&
+ (deployment!.status?.observedGeneration ?? 0) >= deployment!.metadata!.generation! &&
+ (deployment!.status?.updatedReplicas ?? 0) > 0 &&
+ HostReconciler.deploymentReady(deployment)
+ ) {
+ // Aggregate Ready is only possible-use evidence, not an acknowledgement.
+ // Persist it for this UID/revision so a later loss of Ready can recover
+ // once; uninterrupted NotReady startup never acquires this observation.
+ desired[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] = revision
+ }
+ return Object.fromEntries(
+ Object.entries(desired).filter(([key, value]) => current[key] !== value)
+ )
+ }
+
private static credentialRecordBelongsToHost(
record: { metadata?: { labels?: Record<string, string> } },
host: HostCRD
@@ -1647,7 +1732,7 @@ export class HostReconciler {
private static deploymentBelongsToHost(
deployment: k8s.V1Deployment | null,
host: HostCRD
- ): boolean {
+ ): deployment is k8s.V1Deployment {
return (
!!deployment &&
HostReconciler.credentialRecordBelongsToHost(deployment, host) &&
@@ -1699,16 +1784,32 @@ export class HostReconciler {
credentialRecord: k8s.V1Secret | null,
options: BootstrapOptions
): boolean {
- if (options.forceFreshForWake || !deployment) return true
+ if (options.forceFreshForWake || options.forceFreshForDeploymentChange || !deployment)
+ return true
// This reconcile will converge the Deployment to zero, so no Pod will
// consume the stored material before the next verified wake.
if (options.targetSuspended === true) return false
if ((deployment.spec?.replicas ?? 1) === 0) return true
+ const annotations = credentialRecord?.metadata?.annotations ?? {}
+ const fresh =
+ annotations[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] === RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH
+ const boundUid = annotations[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
+ if (boundUid && boundUid !== deployment.metadata?.uid) return true
if (HostReconciler.deploymentReady(deployment)) return false
if (
- credentialRecord?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
- RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH &&
- HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment)
+ fresh &&
+ credentialRecord &&
+ HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) &&
+ annotations[RUNTIME_TOKEN_BOOTSTRAP_READY_REVISION_ANNOTATION] ===
+ HostReconciler.deploymentRuntimeTokenRevision(deployment)
+ )
+ return true
+ if (
+ fresh &&
+ credentialRecord &&
+ (HostReconciler.bootstrapBindingMatchesDeployment(credentialRecord, deployment) ||
+ (boundUid === deployment.metadata?.uid &&
+ annotations[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] === 'true'))
) {
return false
}
@@ -1960,22 +2061,22 @@ export class HostReconciler {
)
return null
}
- // A fresh bootstrap is consumed here when it is bound to this
- // Deployment (a wake bootstrap) or when a Ready Deployment already
- // runs its revision (a normal mint whose pod booted before channel
- // authority was lost). The pod rotates the single-use refresh token
- // it booted with, so leaving the record fresh would later roll a new
- // pod onto that revoked token. The pass returns after the one write;
- // GFS renewal follows on the next hold pass.
- if (
- existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
- RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH &&
- (HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment) ||
- (HostReconciler.deploymentReady(deployment) &&
- HostReconciler.deploymentRuntimeTokenRevision(deployment) === existingRevision))
- ) {
- const appliedRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
- if (HostReconciler.deploymentBootstrapConverged(deployment)) {
+ const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
+ host,
+ existing,
+ deployment,
+ existingRevision ?? ''
+ )
+ const renewalDue = HostReconciler.credentialRenewalWindowReached(
+ existing,
+ nowMs,
+ GFS_TOKEN_REFRESH_BEFORE_ANNOTATION
+ )
+ // A pending strict acknowledgement cannot starve independent GFS
+ // renewal. Bind the applied bootstrap before changing the full
+ // content hash; runtime bytes and the applied revision stay intact.
+ if (!renewalDue || !HostReconciler.deploymentReady(deployment)) {
+ if (Object.keys(annotationUpdates).length > 0) {
await this.coreApi.replaceNamespacedSecret({
name,
namespace: host.namespace,
@@ -1985,28 +2086,17 @@ export class HostReconciler {
...existing.metadata,
annotations: {
...(existing.metadata?.annotations ?? {}),
- [RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]:
- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED,
- [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'false',
+ ...annotationUpdates,
},
},
},
})
}
- return {
- revision: appliedRevision || existingRevision || '',
- scopeHash:
- existing.metadata?.annotations?.[RUNTIME_TOKEN_SCOPE_HASH_ANNOTATION] ?? '',
- }
- }
- if (!HostReconciler.deploymentReady(deployment)) return null
- if (
- !HostReconciler.credentialRenewalWindowReached(
- existing,
- nowMs,
- GFS_TOKEN_REFRESH_BEFORE_ANNOTATION
+ if (
+ !HostReconciler.deploymentReady(deployment) &&
+ !HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
)
- ) {
+ return null
this.recordGfsLifecycleEvidence(host, 'reused')
return {
revision:
@@ -2034,6 +2124,7 @@ export class HostReconciler {
...existing.metadata,
annotations: {
...(existing.metadata?.annotations ?? {}),
+ ...annotationUpdates,
[GFS_TOKEN_EXPECTED_SUBJECT_ANNOTATION]:
makeExpectedHostGfsSubject(host.namespace, host.name) ?? '',
[GFS_TOKEN_CAPABILITY_SET_HASH_ANNOTATION]: HostReconciler.gfsCapabilitySetHash(),
@@ -2125,31 +2216,6 @@ export class HostReconciler {
}
}
- if (
- existingRevision &&
- !decided.refresh &&
- options.prepareForResourceChange &&
- HostReconciler.deploymentBelongsToHost(deployment, host) &&
- HostReconciler.deploymentBootstrapConverged(deployment) &&
- !HostReconciler.deploymentResourcesMatchHost(deployment!, host) &&
- !(
- bootstrapIsFresh &&
- existing?.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] ===
- 'true' &&
- HostReconciler.deploymentRuntimeTokenRevision(deployment) !== existingRevision
- )
- ) {
- // A resource edit replaces the Ready pod and loses its rotated
- // in-memory lineage. Persist usable bootstrap before that replace.
- // A valid fresh pending revision already prepares the same retry;
- // scope, identity and expiry still pass the decision above.
- decided = {
- refresh: true,
- rolloutRequired: true,
- reason: 'fresh_mint_for_resource_change',
- }
- }
-
// Revoked-on-wake guard: the mcp-host runtime refresh token is
// single-use-rotating (control-api revokes the prior JTI on every
// refresh). A pod that is about to boot -- wake (scale 0->1), missing
@@ -2251,26 +2317,76 @@ export class HostReconciler {
projection,
grokProjection
)
- const decision = decideRefresh(observedFrontsOAuth)
+ let decision = decideRefresh(observedFrontsOAuth)
+ const resourcesWillChange =
+ options.prepareForResourceChange === true &&
+ HostReconciler.deploymentBelongsToHost(deployment, host) &&
+ (deployment.spec?.replicas ?? 1) > 0 &&
+ !HostReconciler.deploymentResourcesMatchHost(deployment!, host)
+ const bootstrapApplied =
+ !!existing &&
+ !!existingRevision &&
+ HostReconciler.bootstrapAppliedToDeployment(existing, deployment, existingRevision)
+ const boundUid =
+ existing?.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]
+ const usablePendingPreparation =
+ bootstrapIsFresh &&
+ !bootstrapApplied &&
+ (!boundUid || boundUid === deployment?.metadata?.uid)
+ if (resourcesWillChange) {
+ // Resource preparation is optional until scope authority is observed;
+ // it must not participate in the intrinsic renewal/OAuth fallback above.
+ // Compose a real transition with an already due renewal in one issuance.
+ decision = {
+ ...decision,
+ refresh: decision.refresh || !usablePendingPreparation,
+ rolloutRequired: true,
+ reason: decision.refresh ? decision.reason : 'fresh_mint_for_resource_change',
+ }
+ }
if (existing && existingRevision && !decision.refresh) {
const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
const rolloutMarker =
existing.metadata?.annotations?.[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]
+ const boundTargetRevision =
+ existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]
+ const preparedRevision =
+ bootstrapIsFresh && boundUid === deployment?.metadata?.uid && boundTargetRevision
+ ? boundTargetRevision
+ : existingRevision
const bootstrapBoundToAppliedDeployment =
HostReconciler.bootstrapBindingMatchesDeployment(existing, deployment)
- const bootstrapPendingForAppliedDeployment =
- bootstrapIsFresh && bootstrapBoundToAppliedDeployment
+ const bootstrapPendingForAppliedDeployment = bootstrapIsFresh && bootstrapApplied
const bootstrapConsumedForAppliedDeployment =
existing.metadata?.annotations?.[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] ===
RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED && bootstrapBoundToAppliedDeployment
const rolloutPending =
- rolloutMarker === 'true' ||
- (rolloutMarker === undefined &&
- bootstrapIsFresh &&
- deploymentRevision !== '' &&
- deploymentRevision !== existingRevision)
- const annotationUpdates: Record<string, string> = {}
+ !bootstrapApplied &&
+ (resourcesWillChange ||
+ rolloutMarker === 'true' ||
+ (rolloutMarker === undefined &&
+ bootstrapIsFresh &&
+ deploymentRevision !== '' &&
+ deploymentRevision !== existingRevision))
+ const annotationUpdates = HostReconciler.bootstrapObservationUpdates(
+ host,
+ existing,
+ deployment,
+ existingRevision
+ )
+ if (resourcesWillChange && usablePendingPreparation && !bootstrapApplied) {
+ // Older preparations can already carry the rollout marker without
+ // a target binding. Persist that binding before delivering them.
+ const preparationAnnotations = {
+ [RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: 'true',
+ [RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]: deployment!.metadata!.uid!,
+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preparedRevision,
+ }
+ for (const [key, value] of Object.entries(preparationAnnotations)) {
+ if (existing.metadata?.annotations?.[key] !== value) annotationUpdates[key] = value
+ }
+ }
if (cacheSynced) {
const currentIngress =
existing.metadata?.annotations?.[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]
@@ -2291,23 +2407,6 @@ export class HostReconciler {
) {
annotationUpdates[GFS_TOKEN_HOST_GENERATION_ANNOTATION] = String(host.generation)
}
- if (
- bootstrapPendingForAppliedDeployment &&
- HostReconciler.deploymentBootstrapConverged(deployment)
- ) {
- annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] =
- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED
- annotationUpdates[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] = 'false'
- }
- if (
- bootstrapIsFresh &&
- HostReconciler.deploymentBootstrapConverged(deployment) &&
- deploymentRevision === existingRevision
- ) {
- annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION] =
- RUNTIME_TOKEN_BOOTSTRAP_STATE_CONSUMED
- annotationUpdates[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION] = 'false'
- }
if (Object.keys(annotationUpdates).length > 0) {
await this.coreApi.replaceNamespacedSecret({
name,
@@ -2340,15 +2439,24 @@ export class HostReconciler {
(!bootstrapPendingForAppliedDeployment &&
!bootstrapConsumedForAppliedDeployment &&
HostReconciler.shouldRollForRuntimeSecret(deployment, false))
- ? existingRevision
+ ? preparedRevision
: deploymentRevision || existingRevision
return {
revision: selectedRevision,
scopeHash,
+ deploymentUid: deployment?.metadata?.uid,
+ freshBootstrapPrepared:
+ bootstrapIsFresh &&
+ !bootstrapApplied &&
+ (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION] ?? boundUid) ===
+ deployment?.metadata?.uid &&
+ (annotationUpdates[RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION] ??
+ boundTargetRevision) === selectedRevision,
}
}
if (
+ !resourcesWillChange &&
existing &&
existingRevision &&
(decision.reason === 'refresh_before_reached' ||
@@ -2369,6 +2477,7 @@ export class HostReconciler {
return {
revision: HostReconciler.deploymentRuntimeTokenRevision(deployment) || existingRevision,
scopeHash,
+ deploymentUid: deployment?.metadata?.uid,
}
}
@@ -2411,12 +2520,14 @@ export class HostReconciler {
),
[RUNTIME_TOKEN_BOOTSTRAP_STATE_ANNOTATION]: RUNTIME_TOKEN_BOOTSTRAP_STATE_FRESH,
[RUNTIME_TOKEN_ROLLOUT_REQUIRED_ANNOTATION]: rolloutRequired ? 'true' : 'false',
- ...(preserveWakeTemplate && deployment
+ ...((preserveWakeTemplate || rolloutRequired) &&
+ HostReconciler.deploymentBelongsToHost(deployment, host)
? {
[RUNTIME_TOKEN_BOOTSTRAP_DEPLOYMENT_UID_ANNOTATION]:
deployment.metadata?.uid ?? '',
- [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]:
- HostReconciler.deploymentRuntimeTokenRevision(deployment),
+ [RUNTIME_TOKEN_BOOTSTRAP_APPLIED_REVISION_ANNOTATION]: preserveWakeTemplate
+ ? HostReconciler.deploymentRuntimeTokenRevision(deployment)
+ : revision,
}
: {}),
[RUNTIME_TOKEN_HAS_CHANNEL_INGRESS_ANNOTATION]: String(hasChannelIngress),
@@ -2440,6 +2551,9 @@ export class HostReconciler {
? HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision
: revision,
scopeHash,
+ deploymentUid: deployment?.metadata?.uid,
+ freshBootstrapPrepared:
+ rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
}
}
@@ -2470,6 +2584,9 @@ export class HostReconciler {
? revision
: HostReconciler.deploymentRuntimeTokenRevision(deployment) || revision,
scopeHash,
+ deploymentUid: deployment?.metadata?.uid,
+ freshBootstrapPrepared:
+ rolloutRequired && HostReconciler.deploymentBelongsToHost(deployment, host),
}
} catch (err) {
lastErr = err
@@ -4082,16 +4199,27 @@ export class HostReconciler {
mounts: ResolvedSfsMount[],
runtimeTokenRevision: string | undefined,
lifecycle?: EffectiveHostLifecycle,
- resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>,
+ resolveStateBeforeMutation?: (observed?: k8s.V1Deployment) => Promise<DeploymentMutationState>,
revalidate?: () => void
): Promise<boolean> {
let observedDeployment: k8s.V1Deployment | undefined
let holdingTemplate = false
let applied = true
const buildDesiredDeployment = async (): Promise<k8s.V1Deployment | null> => {
- const state = resolveStateBeforeMutation ? await resolveStateBeforeMutation() : null
+ const state = resolveStateBeforeMutation
+ ? await resolveStateBeforeMutation(observedDeployment)
+ : null
const effective = state?.lifecycle ?? lifecycle
- holdingTemplate = effective?.suspensionBlocked === true
+ const resourcesDiffer =
+ !!observedDeployment &&
+ HostReconciler.deploymentBelongsToHost(observedDeployment, host) &&
+ !HostReconciler.deploymentResourcesMatchHost(observedDeployment, host)
+ const suspendingWithResourceDrift =
+ effective?.stateless === true &&
+ effective.state === 'suspended' &&
+ effective.suspensionBlocked !== true &&
+ resourcesDiffer
+ holdingTemplate = effective?.suspensionBlocked === true || suspendingWithResourceDrift
if (holdingTemplate) {
if (!observedDeployment) return null
const existing = observedDeployment
@@ -4111,22 +4239,50 @@ export class HostReconciler {
if (!hostUidVerified) {
throw new Error(`Cannot preserve an unverified Deployment for Host "${host.name}"`)
}
- // Preserve UID/resourceVersion and every applied field. Conflicts
- // re-read this object before retrying; replicas is the only field that
- // may change. An unverified legacy object must never be scaled.
- const replicas = effective?.allowScaleUpDuringHold ? 1 : (existing.spec.replicas ?? 1)
+ // Preserve UID/resourceVersion and every applied field. A real suspend
+ // must still reach zero when resource policy changes; its next wake
+ // prepares credentials before applying that deferred template change.
+ // Conflicts re-read identity before retrying the replica-only mutation.
+ const replicas = suspendingWithResourceDrift
+ ? 0
+ : effective?.allowScaleUpDuringHold
+ ? 1
+ : (existing.spec.replicas ?? 1)
return {
...existing,
spec: { ...existing.spec, replicas },
}
}
- return this.buildDeployment(
+ if (
+ observedDeployment &&
+ resourcesDiffer &&
+ (observedDeployment.spec?.replicas ?? 1) > 0 &&
+ effective?.state === 'active' &&
+ state?.freshBootstrapPrepared !== true
+ ) {
+ // Readiness cannot gate a resource repair that is needed for scheduling.
+ // Delivery is gated on persisted, fresh material for the observed target.
+ throw new RuntimeBootstrapPreparationUnavailableError()
+ }
+ const desired = this.buildDeployment(
host,
mounts,
state?.runtimeTokenRevision ?? runtimeTokenRevision,
effective,
state?.grokExecutionEnabled ?? this.hostDerivesGrokExecution(host)
)
+ if (observedDeployment && resourcesDiffer && effective?.state === 'draining') {
+ // Defer only resource policy while draining. An intrinsic credential
+ // recovery must still deliver its prepared runtime revision.
+ const appliedResources = observedDeployment.spec!.template.spec!.containers.find(
+ container => container.name === 'mcp-host'
+ )?.resources
+ const desiredContainer = desired.spec!.template.spec!.containers.find(
+ container => container.name === 'mcp-host'
+ )!
+ desiredContainer.resources = structuredClone(appliedResources)
+ }
+ return desired
}
const mutationAllowed = () => {
revalidate?.()
@@ -5575,7 +5731,9 @@ export class HostReconciler {
)
}
- const resolveDeploymentState = async (): Promise<DeploymentMutationState> => {
+ const resolveDeploymentState = async (
+ observed?: k8s.V1Deployment
+ ): Promise<DeploymentMutationState> => {
for (let attempt = 1; attempt <= 3; attempt++) {
revalidateHostMutationBoundary()
const beforeScope = await resolveDeploymentLifecycle()
@@ -5587,6 +5745,28 @@ export class HostReconciler {
}
}
await ensureCurrentRuntimeTokenScope()
+ const deploymentIdentityChanged =
+ !!observed?.metadata?.uid &&
+ !!requireRuntimeTokenProvision().deploymentUid &&
+ observed.metadata.uid !== requireRuntimeTokenProvision().deploymentUid
+ const observedResourceTransition =
+ !!observed &&
+ beforeScope.state === 'active' &&
+ HostReconciler.deploymentBelongsToHost(observed, host) &&
+ (observed.spec?.replicas ?? 1) > 0 &&
+ !HostReconciler.deploymentResourcesMatchHost(observed, host)
+ if (deploymentIdentityChanged || observedResourceTransition) {
+ // A CAS retry or existence read can reveal a new UID or resource drift.
+ // Prepare against this observation before rebuilding the Pod template;
+ // persisted preparation is reused for retries against the same target.
+ runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
+ forceFreshForDeploymentChange: deploymentIdentityChanged,
+ targetSuspended: beforeScope.stateless && beforeScope.state === 'suspended',
+ prepareForResourceChange: beforeScope.state === 'active',
+ })
+ requireRuntimeTokenProvision()
+ revalidateHostMutationBoundary()
+ }
const effective = await resolveDeploymentLifecycle()
const currentOAuth = await this.observeFrontsOAuthServer(host)
// The OAuth lookup can await I/O after the last lifecycle observation.
@@ -5602,6 +5782,7 @@ export class HostReconciler {
return {
lifecycle: effective,
runtimeTokenRevision: requireRuntimeTokenProvision().revision,
+ freshBootstrapPrepared: requireRuntimeTokenProvision().freshBootstrapPrepared,
grokExecutionEnabled: this.hostDerivesGrokExecution(host),
}
}
@@ -5628,12 +5809,19 @@ export class HostReconciler {
revalidateHostMutationBoundary
)
} catch (error) {
- if (error instanceof Error && error.name === 'RuntimeScopeObservationUnavailableError') {
+ if (
+ error instanceof Error &&
+ (error.name === 'RuntimeScopeObservationUnavailableError' ||
+ error.name === 'RuntimeBootstrapPreparationUnavailableError')
+ ) {
const liveDeployment = await this.readHostDeploymentOrNull(host)
this.setStatus(host.name, {
deployed: !!liveDeployment,
ready: false,
- message: 'Waiting for authoritative scope observation',
+ message:
+ error.name === 'RuntimeBootstrapPreparationUnavailableError'
+ ? error.message
+ : 'Waiting for authoritative scope observation',
})
return
}
diff --git a/host-context-controller/src/kubernetesQuantity.test.ts b/host-context-controller/src/kubernetesQuantity.test.ts
new file mode 100644
index 000000000..d7d6e4bc8
--- /dev/null
+++ b/host-context-controller/src/kubernetesQuantity.test.ts
@@ -0,0 +1,156 @@
+import { describe, expect, it } from 'vitest'
+import { kubernetesQuantitiesEqual } from './kubernetesQuantity'
+
+describe('kubernetesQuantitiesEqual', () => {
+ it.each([
+ ['1000m', '1'],
+ ['0.025', '25m'],
+ ['1000n', '1u'],
+ ['1000u', '1m'],
+ ['1k', '1000'],
+ ['1M', '1000k'],
+ ['1G', '1000M'],
+ ['1T', '1000G'],
+ ['1P', '1000T'],
+ ['1E', '1000P'],
+ ['1Ki', '1024'],
+ ['1024Ki', '1Mi'],
+ ['1024Mi', '1Gi'],
+ ['1024Gi', '1Ti'],
+ ['1024Ti', '1Pi'],
+ ['1024Pi', '1Ei'],
+ ['1.5Gi', '1536Mi'],
+ ['100.035Ki', '102435.84'],
+ ['0.025Ti', '27487790694.4'],
+ ['1e3', '1k'],
+ ['1E-3', '1m'],
+ ['1.3E+6', '1300k'],
+ ['025.000e-3', '+25m'],
+ ['.001', '1m'],
+ ['1.', '1'],
+ ['1.G', '1G'],
+ ['-1000m', '-1'],
+ ['-1Gi', '-1024Mi'],
+ ['-0', '+0.000Gi'],
+ ['0e2147483647', '0n'],
+ ])('recognizes %s and %s as equal', (left, right) => {
+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)
+ expect(kubernetesQuantitiesEqual(right, left)).toBe(true)
+ })
+
+ it.each([
+ ['1', '1001m'],
+ ['1k', '1Ki'],
+ ['1M', '1Mi'],
+ ['25m', '0.026'],
+ ['1', '-1'],
+ ['0', '1n'],
+ ['9007199254740992', '9007199254740993'],
+ ['1.000000001', '1'],
+ ['1Gi', '1073741824.000000001'],
+ ['12E', '9223372036854775807'],
+ ])('distinguishes %s and %s', (left, right) => {
+ expect(kubernetesQuantitiesEqual(left, right)).toBe(false)
+ expect(kubernetesQuantitiesEqual(right, left)).toBe(false)
+ })
+
+ it.each([
+ ['3.001n', '4n'],
+ ['1.1E-9', '2n'],
+ ['0.0000000001', '1n'],
+ ['0.00000012345', '124n'],
+ ['0.000000000001Ki', '2n'],
+ ['-0.0000000001', '-1n'],
+ ['-3.001n', '-4n'],
+ ['1.0000000001', '1.000000001'],
+ ['9.9999999991', '10'],
+ ['-9.9999999991', '-10'],
+ ['0.0000000010000', '1n'],
+ ['0.0000000000000', '0'],
+ ])('rounds %s away from zero at nano precision to %s', (left, right) => {
+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)
+ })
+
+ it.each([
+ ['9Ei', '9223372036854775807'],
+ ['9223372036854775807Ki', '9223372036854775807'],
+ ['-9Ei', '-9223372036854775807'],
+ ['7.9999999999999999999Ei', '9223372036854775807'],
+ ['12E', '12000000000000000000'],
+ ['9223372036854775808', '9223372036854775808e0'],
+ ])('applies format-specific maximum rules to %s and %s', (left, right) => {
+ expect(kubernetesQuantitiesEqual(left, right)).toBe(true)
+ })
+
+ it.each([
+ '',
+ '+',
+ '-',
+ '.',
+ '-.',
+ 'm',
+ 'Ki',
+ '+Gi',
+ '1.1.M',
+ '1+1.0M',
+ '0.1mi',
+ '0.1am',
+ 'aoeu',
+ '.5i',
+ '1i',
+ '-3.01e-',
+ '1e',
+ '1K',
+ '1e1.5',
+ ' 1',
+ '1 ',
+ '1\n',
+ 'NaN',
+ 'Infinity',
+ '1e9223372036854775808',
+ '1e-9223372036854775809',
+ ])('rejects invalid quantities with a fixed error (%j)', value => {
+ expect(() => kubernetesQuantitiesEqual(value, '1')).toThrowError(
+ new Error('Invalid Kubernetes quantity')
+ )
+ expect(() => kubernetesQuantitiesEqual('1', value)).toThrowError(
+ new Error('Invalid Kubernetes quantity')
+ )
+ expect(() => kubernetesQuantitiesEqual(value, value)).toThrowError(
+ new Error('Invalid Kubernetes quantity')
+ )
+ })
+
+ it('compares large positive exponents without expanding powers of ten', () => {
+ expect(kubernetesQuantitiesEqual('1e1000000000', '10e999999999')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e1000000000', '1e999999999')).toBe(false)
+ expect(kubernetesQuantitiesEqual('1e2147483647', '10e2147483646')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e-1000000000', '1n')).toBe(true)
+ expect(kubernetesQuantitiesEqual('-1e-1000000000', '-1n')).toBe(true)
+ })
+
+ it('preserves the upstream int64-to-int32 exponent narrowing', () => {
+ expect(kubernetesQuantitiesEqual('1e4294967296', '1')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e4294967295', '0.1')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e9223372036854775807', '0.1')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e-9223372036854775808', '1')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1.0e-2147483648', '10e2147483647')).toBe(true)
+ })
+
+ it('preserves the slow parser nano scale wrapping at int32 boundaries', () => {
+ expect(kubernetesQuantitiesEqual('1000000000000000000e2147483647', '1n')).toBe(true)
+ expect(kubernetesQuantitiesEqual('1e-2147483648', '1n')).toBe(true)
+ })
+
+ it('rejects exponents outside int64 before creating an exponent BigInt', () => {
+ const value = `1e${'9'.repeat(10000)}`
+ expect(() => kubernetesQuantitiesEqual(value, '1')).toThrowError(
+ new Error('Invalid Kubernetes quantity')
+ )
+ })
+
+ it('handles long zero padding without expanding the exponent', () => {
+ expect(kubernetesQuantitiesEqual(`1e+${'0'.repeat(10000)}1`, '10')).toBe(true)
+ expect(kubernetesQuantitiesEqual(`1${'0'.repeat(10000)}e-10000`, '1')).toBe(true)
+ })
+})
diff --git a/host-context-controller/src/kubernetesQuantity.ts b/host-context-controller/src/kubernetesQuantity.ts
new file mode 100644
index 000000000..78d8ac8dc
--- /dev/null
+++ b/host-context-controller/src/kubernetesQuantity.ts
@@ -0,0 +1,89 @@
+const decimalSuffixes = ['n', 'u', 'm', '', 'k', 'M', 'G', 'T', 'P', 'E']
+const binarySuffixes = ['Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei']
+const maximumBinaryQuantity = '9223372036854775807'
+
+function invalidQuantity(): never {
+ throw new Error('Invalid Kubernetes quantity')
+}
+
+function decimalExponent(suffix: string): number {
+ const negative = suffix[1] === '-'
+ const digits = suffix.slice(1).replace(/^[+-]/, '').replace(/^0+/, '') || '0'
+ const maximum = negative ? '9223372036854775808' : '9223372036854775807'
+ if (digits.length > maximum.length || (digits.length === maximum.length && digits > maximum)) {
+ invalidQuantity()
+ }
+ // Apimachinery parses a signed int64 exponent, then narrows it to int32.
+ return Number(BigInt.asIntN(32, BigInt(`${negative ? '-' : ''}${digits}`)))
+}
+
+function normalizedQuantity(value: string): string {
+ const match =
+ /^([+-]?)([0-9]*)(?:\.([0-9]*))?(n|u|m|k|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei|[eE][+-]?[0-9]+)?$/.exec(
+ value
+ )
+ // Require a digit as the Quantity grammar does, rather than accepting the
+ // Go scanner's undocumented bare-sign, bare-dot, or bare-suffix zero aliases.
+ if (!match || match[0] !== value || !(match[2] || match[3])) {
+ invalidQuantity()
+ }
+
+ const numerator = match[2].replace(/^0+/, '') || '0'
+ const fraction = match[3] || ''
+ const suffix = match[4] || ''
+ const binaryIndex = binarySuffixes.indexOf(suffix)
+ const decimalIndex = decimalSuffixes.indexOf(suffix)
+ let digits = `${numerator}${fraction}`.replace(/^0+/, '') || '0'
+ let exponent = -fraction.length
+
+ if (binaryIndex >= 0) {
+ // BinarySI has only six fixed exponents, so this shift cannot grow with an
+ // exponent supplied by the caller.
+ digits = (BigInt(digits) << BigInt((binaryIndex + 1) * 10)).toString()
+ } else {
+ const suffixExponent = decimalIndex >= 0 ? (decimalIndex - 3) * 3 : decimalExponent(suffix)
+ exponent = (suffixExponent - fraction.length) | 0
+ if (numerator.length + fraction.length > 18 || exponent < -9) {
+ // The slow decimal parser rounds to nano using int32 scale arithmetic.
+ // Preserve its scale wrapping without constructing its power of ten.
+ exponent = ((exponent + 9) | 0) - 9
+ }
+ }
+
+ if (digits === '0') return '0'
+
+ if (exponent < -9) {
+ const keptDigits = digits.length + exponent + 9
+ if (keptDigits <= 0) {
+ digits = '1'
+ } else {
+ const remainder = /[1-9]/.test(digits.slice(keptDigits))
+ digits = (BigInt(digits.slice(0, keptDigits)) + (remainder ? 1n : 0n)).toString()
+ }
+ exponent = -9
+ }
+
+ const withoutTrailingZeroes = digits.replace(/0+$/, '')
+ exponent += digits.length - withoutTrailingZeroes.length
+ digits = withoutTrailingZeroes
+
+ // ParseQuantity caps only BinarySI; DecimalSI and exponent quantities can
+ // exceed int64. After nano rounding, this comparison needs at most 28 digits.
+ if (
+ binaryIndex >= 0 &&
+ (digits.length + exponent > maximumBinaryQuantity.length ||
+ (digits.length + exponent === maximumBinaryQuantity.length &&
+ digits.padEnd(maximumBinaryQuantity.length, '0') > maximumBinaryQuantity))
+ ) {
+ digits = maximumBinaryQuantity
+ exponent = 0
+ }
+
+ // Keep powers of ten as an integer exponent, including very large values.
+ return `${match[1] === '-' ? '-' : ''}${digits}e${exponent}`
+}
+
+/** Compare numeric Quantity values using apimachinery v0.34.0 precision rules. */
+export function kubernetesQuantitiesEqual(left: string, right: string): boolean {
+ return normalizedQuantity(left) === normalizedQuantity(right)
+}
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 43e2a1d72..d1cb138bd 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1,5 +1,6 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import * as k8s from '@kubernetes/client-node'
+import { isDeepStrictEqual } from 'node:util'
import { asApiserverDeployment } from '../src/__tests__/asApiserverDeployment'
import type { AdministrativeOutcomeReporter } from '../src/administrativeOutcomeReporter'
import { config } from '../src/config'
@@ -534,8 +535,11 @@ function rejectedCondition(status: HostCrdStatus) {
describe('HostReconciler resource rollout bootstrap', () => {
const initialResources = structuredClone(config.hostResources)
const initialDesktopResources = structuredClone(config.desktopResources)
+ const initialRefreshBefore = config.mcpHostBootstrapRefreshBeforeSec
const issue = vi.mocked(issueMcpHostRuntimeTokens)
const initialIssue = issue.getMockImplementation()!
+ const gfsMint = vi.mocked(mintHostGfsToken)
+ const initialGfsMint = gfsMint.getMockImplementation()!
const BOOTSTRAP = 'clerum.io/runtime-token-bootstrap-state'
const SECRET_REVISION = 'clerum.io/runtime-token-secret-revision'
const APPLIED_REVISION = 'clerum.io/runtime-token-revision'
@@ -545,7 +549,9 @@ describe('HostReconciler resource rollout bootstrap', () => {
afterEach(() => {
config.hostResources = structuredClone(initialResources)
config.desktopResources = structuredClone(initialDesktopResources)
+ config.mcpHostBootstrapRefreshBeforeSec = initialRefreshBefore
issue.mockImplementation(initialIssue)
+ gfsMint.mockImplementation(initialGfsMint)
vi.useRealTimers()
// Full reconciles schedule readiness polls; keep their timers inside this fixture.
for (const reconciler of fixtures.splice(0)) {
@@ -554,15 +560,29 @@ describe('HostReconciler resource rollout bootstrap', () => {
})
async function runtime(
- options: { desktop?: boolean; stateless?: boolean; consumed?: boolean } = {}
+ options: {
+ desktop?: boolean
+ stateless?: boolean
+ draining?: boolean
+ consumed?: boolean
+ oauth?: boolean
+ canonicalResources?: boolean
+ } = {}
) {
vi.useFakeTimers({ toFake: ['Date'] })
vi.setSystemTime(new Date('2026-07-03T00:00:00.000Z'))
+ // A finite producer-derived renewal window exercises maintenance before expiry.
+ config.mcpHostBootstrapRefreshBeforeSec = 90
const host = makeHost({
generation: 1,
...(options.stateless
? makeStatelessHost({
- status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
+ status: {
+ lifecycle: {
+ state: options.draining ? 'draining' : 'active',
+ wakeHandledGeneration: 0,
+ },
+ },
})
: {}),
...(options.desktop ? { spec: { ...makeHost().spec, desktop: { browser: true } } } : {}),
@@ -571,7 +591,7 @@ describe('HostReconciler resource rollout bootstrap', () => {
const apis = createReconciler({ isCommunicationChannelCacheSynced: () => cacheSynced })
const { reconciler, appsApi, coreApi, customApi } = apis
fixtures.push(reconciler)
- reconciler.setHostFrontsOAuthServer(async () => false)
+ reconciler.setHostFrontsOAuthServer(async () => options.oauth ?? false)
const readHost = customApi.getNamespacedCustomObject.getMockImplementation()!
customApi.getNamespacedCustomObject.mockImplementation(async request =>
request.plural === 'hosts'
@@ -583,6 +603,7 @@ describe('HostReconciler resource rollout bootstrap', () => {
)
let issuance = 0
+ let gfsIssuance = 0
const events: string[] = []
// Only the issuer boundary is simulated. Distinct synthetic emissions and
// decodable future refresh expiries exercise the real persistence/reuse path.
@@ -596,10 +617,17 @@ describe('HostReconciler resource rollout bootstrap', () => {
mcpHostControlToken: `synthetic-control-${emission}`,
}
})
+ // Independent synthetic GFS emissions expose content-hash changes without
+ // changing the runtime access/refresh/control material.
+ gfsMint.mockImplementation(async (...args) => ({
+ ...(await initialGfsMint(...args)),
+ token: `synthetic-gfs-${++gfsIssuance}`,
+ }))
let secret: k8s.V1Secret | null = null
let deployment: k8s.V1Deployment | null = null
let secretVersion = 0
let deploymentVersion = 0
+ let templateChanges = 0
let replacementFailures: number[] = []
let secretConflict = false
const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
@@ -653,6 +681,31 @@ describe('HostReconciler resource rollout bootstrap', () => {
})
const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ const persistDeployment = (body: k8s.V1Deployment) => {
+ const result = asApiserverDeployment(body)
+ // The recorded-default overlay does not merge Date objects. A real SDK
+ // read preserves these timestamps on a replica-only replacement.
+ for (const [incoming, persisted] of [
+ [body.metadata, result.metadata],
+ [body.spec?.template.metadata, result.spec?.template.metadata],
+ ]) {
+ if (incoming?.creationTimestamp instanceof Date && persisted) {
+ persisted.creationTimestamp = structuredClone(incoming.creationTimestamp)
+ }
+ }
+ if (options.canonicalResources) {
+ // Closed equivalences from apimachinery Quantity parse/serialization,
+ // independent of the production guard's implementation.
+ for (const container of result.spec!.template.spec!.containers) {
+ for (const kind of ['requests', 'limits'] as const) {
+ const values = container.resources?.[kind]
+ if (values?.cpu === '1000m') values.cpu = '1'
+ if (values?.memory === '1024Mi') values.memory = '1Gi'
+ }
+ }
+ }
+ return result
+ }
appsApi.readNamespacedDeployment.mockImplementation(async request => {
if (request.name !== host.name) return readDeployment(request)
if (!deployment) throw { code: 404 }
@@ -663,7 +716,7 @@ describe('HostReconciler resource rollout bootstrap', () => {
const { body } = request
if (body.metadata?.name !== host.name) return createDeployment(request)
if (deployment) throw { code: 409 }
- deployment = asApiserverDeployment(body)
+ deployment = persistDeployment(body)
deployment.metadata = {
...deployment.metadata,
uid: 'runtime-deployment-uid',
@@ -686,13 +739,16 @@ describe('HostReconciler resource rollout bootstrap', () => {
throw { code: failure }
}
const previous = deployment
- deployment = asApiserverDeployment(body)
+ deployment = persistDeployment(body)
deployment.metadata = {
...deployment.metadata,
uid: previous.metadata!.uid,
resourceVersion: String(++deploymentVersion),
- generation: previous.metadata!.generation! + 1,
+ generation:
+ previous.metadata!.generation! +
+ (isDeepStrictEqual(previous.spec, deployment.spec) ? 0 : 1),
}
+ if (!isDeepStrictEqual(previous.spec!.template, deployment.spec!.template)) templateChanges++
// Kubernetes status belongs to the prior generation until advance() is called.
deployment.status = structuredClone(previous.status)
events.push('replace-deployment')
@@ -719,6 +775,7 @@ describe('HostReconciler resource rollout bootstrap', () => {
expect(secret!.metadata!.annotations![BOOTSTRAP]).toBe('consumed')
}
events.length = 0
+ templateChanges = 0
issue.mockClear()
return {
...apis,
@@ -734,9 +791,20 @@ describe('HostReconciler resource rollout bootstrap', () => {
conflictSecret: () => {
secretConflict = true
},
+ unbindLegacyPreparation: () => {
+ delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
+ delete secret!.metadata!.annotations!['clerum.io/runtime-token-bootstrap-applied-revision']
+ secret!.metadata!.resourceVersion = String(++secretVersion)
+ },
setCacheSynced: (value: boolean) => {
cacheSynced = value
},
+ templateChanges: () => templateChanges,
+ recreateDeployment: () => {
+ deployment!.metadata!.uid = 'recreated-runtime-deployment-uid'
+ deployment!.metadata!.resourceVersion = String(++deploymentVersion)
+ deployment!.metadata!.generation = 1
+ },
}
}
@@ -907,6 +975,361 @@ describe('HostReconciler resource rollout bootstrap', () => {
await fixture.reconcile()
expect(issue).toHaveBeenCalledOnce()
})
+
+ it.each(['Desktop CPU', 'Host memory'])(
+ 'keeps equivalent canonical API quantities stable: %s',
+ async kind => {
+ const desktop = kind === 'Desktop CPU'
+ if (!desktop) config.hostResources.requests.memory = '1024Mi'
+ const fixture = await runtime({ desktop, canonicalResources: true, consumed: false })
+ for (let pass = 0; pass < 3; pass++) {
+ await fixture.reconcile()
+ fixture.advance()
+ }
+ expect(issue).not.toHaveBeenCalled()
+ expect(fixture.templateChanges()).toBe(0)
+ expect(fixture.deployment().metadata!.generation).toBe(1)
+ }
+ )
+
+ it('defers resource preparation when retained OAuth authority is temporarily unobserved', async () => {
+ const fixture = await runtime({ oauth: true })
+ config.hostResources.requests.cpu = '25m'
+ fixture.reconciler.setHostFrontsOAuthServer(async () => {
+ throw new Error('simulated OAuth observation unavailable')
+ })
+ await fixture.reconcile()
+ expect(issue).not.toHaveBeenCalled()
+ expect(fixture.events).toEqual([])
+ expect(
+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-fronts-oauth-server']
+ ).toBe('true')
+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
+ '100m'
+ )
+ fixture.reconciler.setHostFrontsOAuthServer(async () => true)
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(issue.mock.calls[0][2]).toContain(OAUTH_USER_TOKEN_SCOPE)
+ expect(fixture.templateChanges()).toBe(1)
+ })
+
+ it('reuses an intrinsic renewal persisted before a failed resource replacement', async () => {
+ const fixture = await runtime()
+ vi.setSystemTime(
+ Date.parse(
+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-refresh-before']
+ ) + 1
+ )
+ config.hostResources.requests.cpu = '25m'
+ fixture.failReplacements(503)
+ await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })
+ const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ prepared
+ )
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('binds and delivers legacy unbound preparation even when its rollout marker is already true', async () => {
+ const fixture = await runtime()
+ config.hostResources.requests.cpu = '25m'
+ fixture.failReplacements(503)
+ await expect(fixture.reconcile()).rejects.toMatchObject({ code: 503 })
+ // The previously shipped preparation did not bind its target until acknowledgement.
+ fixture.unbindLegacyPreparation()
+ fixture.events.length = 0
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.events).toEqual(['persist-secret', 'replace-deployment'])
+ expect(
+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
+ ).toBe(fixture.deployment().metadata!.uid)
+ for (let pass = 0; pass < 3; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('applies and acknowledges the preparation when intrinsic renewal and resources coincide', async () => {
+ const fixture = await runtime()
+ vi.setSystemTime(
+ Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-refresh-before']) + 1
+ )
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ const prepared = fixture.secret().metadata!.annotations![SECRET_REVISION]
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ prepared
+ )
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('renews held GFS while strict bootstrap acknowledgement remains pending', async () => {
+ const fixture = await runtime({ stateless: true, consumed: false })
+ const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]
+ const runtimeData = { ...fixture.secret().data }
+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
+ fixture.setCacheSynced(false)
+ gfsMint.mockClear()
+ vi.setSystemTime(Date.now() + 550_000)
+ await fixture.reconcile()
+ await fixture.reconcile()
+ vi.setSystemTime(Date.now() + 100_000)
+ await fixture.reconcile()
+ expect(gfsMint).toHaveBeenCalledOnce()
+ expect(issue).not.toHaveBeenCalled()
+ expect(
+ Date.parse(fixture.secret().metadata!.annotations!['clerum.io/gfs-token-expires-at'])
+ ).toBeGreaterThan(Date.now())
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
+ for (const key of [
+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
+ ]) {
+ expect(fixture.secret().data![key]).toBe(runtimeData[key])
+ }
+ expect(fixture.secret().data![MCP_HOST_GFS_TOKEN_SECRET_KEY]).not.toBe(
+ runtimeData[MCP_HOST_GFS_TOKEN_SECRET_KEY]
+ )
+ fixture.setCacheSynced(true)
+ await fixture.reconcile()
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ applied
+ )
+ expect(fixture.templateChanges()).toBe(0)
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('recovers once after an observed partial Ready bootstrap subsequently loses Ready', async () => {
+ const fixture = await runtime({ consumed: false })
+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
+ fixture.advance({ readyReplicas: 0, availableReplicas: 0, updatedReplicas: 0 })
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.templateChanges()).toBe(1)
+ })
+
+ it('does not mistake stale-generation Ready for possible bootstrap use during slow startup', async () => {
+ const fixture = await runtime({ consumed: false })
+ fixture.advance({ observedGeneration: 0, updatedReplicas: 0, readyReplicas: 1 })
+ await fixture.reconcile()
+ fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })
+ for (let pass = 0; pass < 5; pass++) await fixture.reconcile()
+ expect(issue).not.toHaveBeenCalled()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
+ expect(fixture.templateChanges()).toBe(0)
+ })
+
+ it('prepares for a changed Deployment UID observed after credential persistence', async () => {
+ const fixture = await runtime()
+ config.hostResources.requests.cpu = '25m'
+ const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!
+ let reads = 0
+ fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {
+ if (request.name === fixture.host.name && ++reads === 2) fixture.recreateDeployment()
+ return read(request)
+ })
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledTimes(2)
+ expect(
+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
+ ).toBe('recreated-runtime-deployment-uid')
+ expect(fixture.templateChanges()).toBe(1)
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledTimes(2)
+ })
+
+ it('does not reuse a consumed record bound to a previous Deployment UID', async () => {
+ const fixture = await runtime()
+ fixture.recreateDeployment()
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(
+ fixture.secret().metadata!.annotations!['clerum.io/runtime-token-bootstrap-deployment-uid']
+ ).toBe('recreated-runtime-deployment-uid')
+ await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ })
+
+ it('replans resource preparation against drift observed at the mutation read with the same UID', async () => {
+ const fixture = await runtime()
+ const read = fixture.appsApi.readNamespacedDeployment.getMockImplementation()!
+ let reads = 0
+ fixture.appsApi.readNamespacedDeployment.mockImplementation(async request => {
+ if (request.name === fixture.host.name && ++reads === 2) {
+ fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'
+ fixture.deployment().metadata!.generation!++
+ fixture.advance()
+ }
+ return read(request)
+ })
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
+ '100m'
+ )
+ })
+
+ it('prepares a consumed-bootstrap resource transition without acknowledging a partial Ready runtime', async () => {
+ const fixture = await runtime()
+ fixture.advance({ replicas: 2, updatedReplicas: 1, readyReplicas: 1 })
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
+ for (let pass = 0; pass < 3; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.templateChanges()).toBe(1)
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('fresh')
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('prepares an immediate resource repair for a fresh applied bootstrap without Ready replicas', async () => {
+ const fixture = await runtime({ consumed: false })
+ fixture.advance({ updatedReplicas: 0, readyReplicas: 0, availableReplicas: 0 })
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ fixture.secret().metadata!.annotations![SECRET_REVISION]
+ )
+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.templateChanges()).toBe(1)
+ fixture.advance()
+ await fixture.reconcile()
+ expect(fixture.secret().metadata!.annotations![BOOTSTRAP]).toBe('consumed')
+ })
+
+ it('recovers a NotReady consumed bootstrap when its resources also drift', async () => {
+ const fixture = await runtime()
+ fixture.advance({ readyReplicas: 0, availableReplicas: 0 })
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
+ '25m'
+ )
+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.templateChanges()).toBe(1)
+ })
+
+ it('replans same-UID resource drift introduced by a conflicting Deployment writer', async () => {
+ config.desktopResources.requests.cpu = '1000m'
+ const fixture = await runtime({ desktop: true, canonicalResources: true })
+ const replace = fixture.appsApi.replaceNamespacedDeployment.getMockImplementation()!
+ let changed = false
+ fixture.appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
+ if (request.name === fixture.host.name && !changed) {
+ changed = true
+ fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu = '50m'
+ fixture.deployment().metadata!.generation!++
+ fixture.advance()
+ // The external writer's new resourceVersion makes this first PUT conflict.
+ fixture.events.push('replace-409')
+ }
+ return replace(request)
+ })
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['replace-409', 'issue', 'persist-secret', 'replace-deployment'])
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources!.requests!.cpu).toBe(
+ '1'
+ )
+ expect(fixture.templateChanges()).toBe(1)
+ })
+
+ it('defers resource transitions while the owned stateless runtime is draining', async () => {
+ const fixture = await runtime({ stateless: true, draining: true })
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(issue).not.toHaveBeenCalled()
+ expect(fixture.events).toEqual([])
+ expect(fixture.templateChanges()).toBe(0)
+ expect(fixture.deployment().spec!.replicas).toBe(1)
+ })
+
+ it('delivers intrinsic bootstrap recovery while draining and preserves the applied resources', async () => {
+ const fixture = await runtime({ stateless: true, draining: true })
+ const resources = structuredClone(
+ fixture.deployment().spec!.template.spec!.containers[0].resources
+ )
+ fixture.advance({ readyReplicas: 0, availableReplicas: 0 })
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(fixture.events).toEqual(['issue', 'persist-secret', 'replace-deployment'])
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ fixture.secret().metadata!.annotations![SECRET_REVISION]
+ )
+ expect(fixture.deployment().spec!.template.spec!.containers[0].resources).toEqual(resources)
+ for (let pass = 0; pass < 4; pass++) await fixture.reconcile()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(fixture.templateChanges()).toBe(1)
+ })
+
+ it('suspends without applying resource drift or resurrecting the owned stateless runtime', async () => {
+ const fixture = await runtime({ stateless: true })
+ const template = structuredClone(fixture.deployment().spec!.template)
+ const runtimeData = { ...fixture.secret().data }
+ fixture.host.status = suspendedStatus()
+ config.hostResources.requests.cpu = '25m'
+ await fixture.reconcile()
+ expect(fixture.deployment().spec!.replicas).toBe(0)
+ expect(fixture.deployment().spec!.template).toEqual(template)
+ expect(fixture.secret().data).toEqual(runtimeData)
+ expect(issue).not.toHaveBeenCalled()
+ expect(fixture.events).toEqual(['replace-deployment'])
+ await fixture.reconcile()
+ expect(fixture.deployment().spec!.replicas).toBe(0)
+ expect(fixture.events).toEqual(['replace-deployment'])
+ expect(issue).not.toHaveBeenCalled()
+ })
+
+ it('keeps consumed runtime credentials and the applied revision stable after held GFS renewal', async () => {
+ const fixture = await runtime({ stateless: true })
+ const applied = fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]
+ const before = { ...fixture.secret().data }
+ fixture.setCacheSynced(false)
+ gfsMint.mockClear()
+ vi.setSystemTime(Date.now() + 550_000)
+ await fixture.reconcile()
+ expect(gfsMint).toHaveBeenCalledOnce()
+ expect(issue).not.toHaveBeenCalled()
+ for (const key of [
+ MCP_HOST_RUNTIME_TOKEN_SECRET_ACCESS_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_REFRESH_KEY,
+ MCP_HOST_RUNTIME_TOKEN_SECRET_CONTROL_KEY,
+ ])
+ expect(fixture.secret().data![key]).toBe(before[key])
+ fixture.setCacheSynced(true)
+ await fixture.reconcile()
+ expect(fixture.deployment().spec!.template.metadata!.annotations![APPLIED_REVISION]).toBe(
+ applied
+ )
+ expect(fixture.templateChanges()).toBe(0)
+ expect(issue).not.toHaveBeenCalled()
+ })
})
describe('HostReconciler stateless lifecycle — buildDeployment replicas', () => {
@@ -2203,12 +2626,18 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
it('does not let a failed live probe narrow a retained OAuth grant before its renewal window', async () => {
const host = makeStatelessHost()
- const { reconciler, coreApi } = createReconciler({
+ const { reconciler, coreApi, appsApi } = createReconciler({
isCommunicationChannelCacheSynced: () => true,
})
- coreApi.readNamespacedSecret.mockResolvedValue(
- await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })
- )
+ const record = await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })
+ coreApi.readNamespacedSecret.mockResolvedValue(record)
+ const running = trustedRuntimeDeployment(reconciler, host)
+ running.spec!.template.metadata!.annotations = {
+ ...running.spec!.template.metadata!.annotations,
+ 'clerum.io/runtime-token-revision':
+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
+ }
+ appsApi.readNamespacedDeployment.mockResolvedValue(running)
const oauthResolver = vi.fn(async (): Promise<boolean> => {
throw new Error('oauth lookup unavailable')
})
@@ -2870,7 +3299,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
return deployment
}
- it('marks a bootstrap bound to the applied Deployment consumed without minting', async () => {
+ it('acknowledges a bound bootstrap and renews due GFS in one CAS write', async () => {
const host = makeStatelessHost()
const bound = await renewHeldGfs({
@@ -2883,7 +3312,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(bound.recordReads).toBeGreaterThan(0)
expect(bound.recordWrites).toBe(1)
expect(bound.provision.revision).toBe(APPLIED_REVISION)
- expect(bound.mints).toBe(0)
+ expect(bound.mints).toBe(1)
})
it('renews once the bootstrap is bound to a different Deployment UID', async () => {
@@ -2893,7 +3322,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
record: freshBootstrapBoundTo(`deployment-${host.name}`),
deployment: appliedDeployment,
})
- expect(bound.mints).toBe(0)
+ expect(bound.mints).toBe(1)
expect(bound.recordWrites).toBe(1)
const replaced = await renewHeldGfs({
@@ -2970,10 +3399,9 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
vi.setSystemTime(t0 + 602_000)
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
- // Liveness witness: the hold renewed GFS once and wrote the record twice
- // (the consumed marker, then the renewed GFS credential).
+ // The independent renewal and strict acknowledgement share one CAS write.
expect(vi.mocked(mintHostGfsToken)).toHaveBeenCalledOnce()
- expect(coreApi.replaceNamespacedSecret.mock.calls.length).toBeGreaterThanOrEqual(2)
+ expect(coreApi.replaceNamespacedSecret.mock.calls.length).toBe(1)
expect(record.metadata!.annotations!['clerum.io/runtime-token-rollout-required']).not.toBe(
'true'
)