Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): defer instead of downgrading oauth:user-token when the McpServer read is unobservable
Alfredo Lopez2327789ed5e6
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 3923e9bb0..4792a9463 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.370",+ "version": "0.1.371","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.370",+ "version": "0.1.371","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex a3198cbcc..4e4821e70 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.370",+ "version": "0.1.371","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 6c3c316da..4ed7020ac 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -708,46 +708,20 @@ export class HostReconciler {/*** Whether this Host fronts an enabled `auth.type: oauth` mcp-server, gating- * the derive-only `oauth:user-token` runtime scope. Fails CLOSED (false) when- * the cross-CRD read throws: an uncertain read must never grant the scope.+ * the derive-only `oauth:user-token` runtime scope. A read that failed after+ * its retries is reported as unobserved instead of collapsing to `false`:+ * "scope unknown" is not "scope revoked". Callers decide what an unknown+ * answer means — the issuance path fails closed only where that cannot strip+ * a grant the running pod holds, and the deployment drift guard+ * (`currentScopeMatchesProvision`) never forces a re-mint on it. An uncertain+ * read never grants the scope.** Each call is an INDEPENDENT live point-in-time read (mirrors* `hasChannelIngress`); the value is deliberately not cached across a- * reconcile. Callers that need one consistent value within a unit of work- * resolve it once and thread the same bool (see the issuance path, which- * threads it into the mint-scope derive and the drift hash so they never- * diverge); the deployment drift guard re-reads it on purpose to catch a- * scope change mid-reconcile.- */- private async frontsOAuthServer(host: HostCRD): Promise<boolean> {- const observation = await this.observeFrontsOAuthServer(host)- if (observation.observed) return observation.frontsOAuthServer- // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every- // attempt throws) still fails CLOSED here and can flip the scope off. That is- // correct — a sustained outage is a genuine "scope unknown" situation, not a- // blip — the retry only smooths over transient blips, never a real outage.- log.warn(- 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',- {- host: host.name,- namespace: host.namespace,- contextRef: host.spec.contextRef,- attempts: observation.attempts,- err:- observation.error instanceof Error- ? observation.error.message- : String(observation.error),- }- )- return false- }-- /**- * The live McpServer-backed OAuth observation behind `frontsOAuthServer`,- * reporting a read that failed after its retries instead of collapsing it to- * `false`. The CommunicationChannel-loss issuance path needs that distinction:- * it has no authoritative channel inventory, so it must not replace an- * unknown OAuth answer with a retained one that could re-grant a revoked scope.+ * reconcile. The issuance path resolves it once and threads the same bool+ * into the mint-scope derive and the drift hash so they never diverge; the+ * deployment drift guard re-reads it on purpose to catch a scope change+ * mid-reconcile.*/private async observeFrontsOAuthServer(host: HostCRD@@ -1831,6 +1805,28 @@ export class HostReconciler {)}+ /**+ * Deployment drift guard: does the provisioned credential still match the+ * Host's current scope contract? An OAuth read that failed after its retries+ * is "scope unknown", so it matches a provision minted under either OAuth+ * answer instead of collapsing to `false` and forcing a re-mint that would+ * strip oauth:user-token and roll the Deployment. The issuance path owns the+ * decision to narrow the grant (see `ensureMcpHostRuntimeTokenSecret`).+ */+ private currentScopeMatchesProvision(+ host: HostCRD,+ provision: RuntimeTokenProvision,+ live: Awaited<ReturnType<HostReconciler['observeFrontsOAuthServer']>>+ ): boolean {+ if (live.observed) {+ return provision.scopeHash === this.runtimeScopeHashFor(host, live.frontsOAuthServer)+ }+ return (+ provision.scopeHash === this.runtimeScopeHashFor(host, true) ||+ provision.scopeHash === this.runtimeScopeHashFor(host, false)+ )+ }+private async refreshCodexSnapshot(): Promise<void> {try {const cm = await this.coreApi.readNamespacedConfigMap({@@ -2050,6 +2046,78 @@ export class HostReconciler {)return null}+ const projection = this.projectCodexForHost(host)+ const grokProjection = this.projectGrokForHost(host)+ const decideRefresh = (+ frontsOAuth: boolean+ ): {+ refresh: boolean+ rolloutRequired: boolean+ reason: string+ refreshTokenExpMs?: number+ } => {+ let decided: {+ refresh: boolean+ rolloutRequired: boolean+ reason: string+ refreshTokenExpMs?: number+ } = existing+ ? HostReconciler.runtimeTokenRefreshDecision(+ host,+ existing,+ nowMs,+ hasChannelIngress,+ frontsOAuth,+ projection,+ grokProjection+ )+ : { refresh: true, rolloutRequired: false, reason: 'missing_secret' }+ if (+ existing &&+ HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)+ ) {+ // A manual restart after the runtime Secret was issued starts pods with+ // env-var bootstrap tokens from that older Secret. Force a re-issue and+ // rollout so every restarted pod observes a fresh, unconsumed token pair.+ decided = {+ refresh: true,+ rolloutRequired: true,+ reason: 'deployment_restarted_after_secret_issued',+ }+ }++ // Revoked-on-wake guard: the mcp-host runtime refresh token is+ // single-use-rotating (control-api revokes the prior JTI on every+ // refresh). A pod that is about to boot -- wake (scale 0->1), missing+ // Deployment, or a not-Ready (re)starting pod -- reads its bootstrap+ // refresh token from THIS Secret. If the pre-suspend pod already+ // rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"+ // hands the booting pod a dead token and readiness 503s forever. When+ // a new pod will consume the Secret, force a fresh mint + rollout even+ // though the decoded exp is far out. Reuse stays correct only for a+ // running, Ready pod that never re-reads the Secret (churn fix).+ if (+ existingRevision &&+ decided.refresh === false &&+ HostReconciler.deploymentNeedsFreshBootstrap(+ deployment,+ existingRevision,+ existing,+ options+ )+ ) {+ decided = {+ refresh: true,+ rolloutRequired: true,+ reason: 'fresh_mint_for_booting_pod',+ ...(decided.refreshTokenExpMs !== undefined+ ? { refreshTokenExpMs: decided.refreshTokenExpMs }+ : {}),+ }+ }+ return decided+ }+// Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and// codex projection into the mint-scope derive, the refresh decision, the// scope hash and the stored annotation, so the minted token and the drift@@ -2059,37 +2127,59 @@ export class HostReconciler {// on the CommunicationChannel cache, so it stays live during channel// cache loss: an admin who removed the OAuth mcp-server while the channel// watch was down must not have `oauth:user-token` re-granted from the- // retained annotation on a held wake. The retained value is used only- // when the live read failed after its retries and it cannot widen the- // grant (`false`); otherwise the mint is skipped (fail closed).+ // retained annotation on a held wake.+ //+ // A live read that failed after its retries is "scope unknown", not+ // "scope revoked". It resolves to `false` (fail closed) only when that+ // cannot narrow a grant the running pod holds: the retained value is+ // `false`, or there is no trusted retained value while channel authority+ // is synced. A retained `true` under synced authority defers the decision+ // while the credential needs no renewal, so an McpServer relist does not+ // strip oauth:user-token and roll the Deployment; once the credential must+ // be renewed, the renewal mints without the unobserved scope. Under channel+ // cache loss the mint is skipped instead.+ const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)let observedFrontsOAuth: boolean- if (cacheSynced) {- observedFrontsOAuth = await this.frontsOAuthServer(host)+ if (liveFrontsOAuth.observed) {+ observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer} else {- const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)- if (liveFrontsOAuth.observed) {- observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer- } else if (retainedFrontsOAuth === false) {+ const observationFailure = {+ host: host.name,+ namespace: host.namespace,+ observation: 'frontsOAuthServer',+ attempts: liveFrontsOAuth.attempts,+ err:+ liveFrontsOAuth.error instanceof Error+ ? liveFrontsOAuth.error.message+ : String(liveFrontsOAuth.error),+ }+ if (retainedFrontsOAuth === false || (cacheSynced && retainedFrontsOAuth !== true)) {+ log.warn(+ 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',+ observationFailure+ )+ observedFrontsOAuth = false+ } else if (cacheSynced) {+ if (!decideRefresh(true).refresh) {+ log.warn(+ 'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',+ observationFailure+ )+ return null+ }+ log.warn(+ 'renewing runtime token without oauth:user-token: OAuth observation unavailable when the credential must be renewed',+ observationFailure+ )observedFrontsOAuth = false} else {log.warn('skipping runtime token mint during channel cache loss without an authoritative OAuth observation',- {- host: host.name,- namespace: host.namespace,- observation: 'frontsOAuthServer',- attempts: liveFrontsOAuth.attempts,- err:- liveFrontsOAuth.error instanceof Error- ? liveFrontsOAuth.error.message- : String(liveFrontsOAuth.error),- }+ observationFailure)return null}}- const projection = this.projectCodexForHost(host)- const grokProjection = this.projectGrokForHost(host)const scopeHash = HostReconciler.runtimeTokenScopeHash(host,hasChannelIngress,@@ -2097,65 +2187,7 @@ export class HostReconciler {projection,grokProjection)- let decision: {- refresh: boolean- rolloutRequired: boolean- reason: string- refreshTokenExpMs?: number- } = existing- ? HostReconciler.runtimeTokenRefreshDecision(- host,- existing,- nowMs,- hasChannelIngress,- observedFrontsOAuth,- projection,- grokProjection- )- : { refresh: true, rolloutRequired: false, reason: 'missing_secret' }- if (- existing &&- HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)- ) {- // A manual restart after the runtime Secret was issued starts pods with- // env-var bootstrap tokens from that older Secret. Force a re-issue and- // rollout so every restarted pod observes a fresh, unconsumed token pair.- decision = {- refresh: true,- rolloutRequired: true,- reason: 'deployment_restarted_after_secret_issued',- }- }-- // Revoked-on-wake guard: the mcp-host runtime refresh token is- // single-use-rotating (control-api revokes the prior JTI on every- // refresh). A pod that is about to boot -- wake (scale 0->1), missing- // Deployment, or a not-Ready (re)starting pod -- reads its bootstrap- // refresh token from THIS Secret. If the pre-suspend pod already- // rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"- // hands the booting pod a dead token and readiness 503s forever. When- // a new pod will consume the Secret, force a fresh mint + rollout even- // though the decoded exp is far out. Reuse stays correct only for a- // running, Ready pod that never re-reads the Secret (churn fix).- if (- existingRevision &&- decision.refresh === false &&- HostReconciler.deploymentNeedsFreshBootstrap(- deployment,- existingRevision,- existing,- options- )- ) {- decision = {- refresh: true,- rolloutRequired: true,- reason: 'fresh_mint_for_booting_pod',- ...(decision.refreshTokenExpMs !== undefined- ? { refreshTokenExpMs: decision.refreshTokenExpMs }- : {}),- }- }+ const decision = decideRefresh(observedFrontsOAuth)if (existing && existingRevision && !decision.refresh) {const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)@@ -5411,8 +5443,10 @@ export class HostReconciler {const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {for (let attempt = 1; attempt <= 3; attempt++) {revalidateHostMutationBoundary()- const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))- if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) return+ const currentOAuth = await this.observeFrontsOAuthServer(host)+ if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {+ return+ }runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {forceFreshForWake: false,@@ -5425,11 +5459,16 @@ export class HostReconciler {throw error}revalidateHostMutationBoundary()- const postProvisionScopeHash = this.runtimeScopeHashFor(- host,- await this.frontsOAuthServer(host)- )- if (requireRuntimeTokenProvision().scopeHash === postProvisionScopeHash) return+ const postProvisionOAuth = await this.observeFrontsOAuthServer(host)+ if (+ this.currentScopeMatchesProvision(+ host,+ requireRuntimeTokenProvision(),+ postProvisionOAuth+ )+ ) {+ return+ }log.warn('CommunicationChannel scope contract changed during token provisioning', {host: host.name,@@ -5455,7 +5494,7 @@ export class HostReconciler {}await ensureCurrentRuntimeTokenScope()const effective = await resolveDeploymentLifecycle()- const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))+ const currentOAuth = await this.observeFrontsOAuthServer(host)// The OAuth lookup can await I/O after the last lifecycle observation.// Retry assessment rather than committing a now-obsolete mode.if (@@ -5464,7 +5503,7 @@ export class HostReconciler {) {continue}- if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) {+ if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {revalidateHostMutationBoundary()return {lifecycle: effective,diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 8a600283a..12abf3866 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -1735,12 +1735,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(runtimeTokenProvision(host))- vi.spyOn(reconciler as any, 'frontsOAuthServer').mockImplementation(async () => {- if (++lookup === 2) synced = false- return false+ // The final lookup loses channel authority and its McpServer read fails too:+ // neither an unobservable OAuth answer nor the lost authority may commit+ // the template.+ reconciler.setHostFrontsOAuthServer(async () => {+ if (++lookup === 1) return false+ synced = false+ throw new Error('mcp-server watch retired')})await reconciler.reconcile(host)- expect(lookup).toBe(2)+ // One observed lookup, then the final lookup's three attempts.+ expect(lookup).toBe(4)expect(live().spec!.template).toEqual(applied.spec!.template)expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe('CommunicationChannelCacheUnsynced'@@ -1808,7 +1813,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })})- it('does not let a retained OAuth observation override a failed live probe', async () => {+ it('does not let a failed live probe narrow a retained OAuth grant before its renewal window', async () => {const host = makeStatelessHost()const { reconciler, coreApi } = createReconciler({isCommunicationChannelCacheSynced: () => true,@@ -1816,21 +1821,21 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {coreApi.readNamespacedSecret.mockResolvedValue(await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true }))- reconciler.setHostFrontsOAuthServer(async () => {+ const oauthResolver = vi.fn(async (): Promise<boolean> => {throw new Error('oauth lookup unavailable')})+ reconciler.setHostFrontsOAuthServer(oauthResolver)vi.mocked(issueMcpHostRuntimeTokens).mockClear()const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)- expect(provision).not.toBeNull()- expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).not.toContain(- OAUTH_USER_TOKEN_SCOPE- )- const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret- expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(- 'false'- )+ // Liveness witness: the record was read and the live probe ran all its attempts.+ expect(coreApi.readNamespacedSecret).toHaveBeenCalled()+ expect(oauthResolver).toHaveBeenCalledTimes(3)+ expect(provision).toBeNull()+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()+ expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()})it('uses the retained channel observation but a live OAuth observation while channel authority is unavailable', async () => {@@ -1865,6 +1870,166 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {})})+ describe('McpServer OAuth observation unavailable while channel authority is synced', () => {+ /**+ * A Host whose Ready pod runs a consumed credential that grants+ * oauth:user-token, over stateful Secret and Deployment mocks.+ */+ async function oauthGrantedRuntime(annotations: Record<string, string> = {}) {+ const host = makeHost()+ const { reconciler, appsApi, coreApi } = createReconciler()+ let record = await mintedRuntimeCredentialRecord(host, {+ frontsOAuthServer: true,+ annotations,+ })+ const secretName = record.metadata!.name!+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!+ coreApi.readNamespacedSecret.mockImplementation(request =>+ request.name === secretName ? Promise.resolve(structuredClone(record)) : readSecret(request)+ )+ coreApi.replaceNamespacedSecret.mockImplementation(async request => {+ if (request.name === secretName) record = structuredClone(request.body as k8s.V1Secret)+ return request.body+ })+ let live = trustedRuntimeDeployment(reconciler, host)+ live.spec!.template!.metadata!.annotations = {+ ...live.spec!.template!.metadata!.annotations,+ 'clerum.io/runtime-token-revision':+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],+ }+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ request.name === host.name+ ? Promise.resolve(structuredClone(live))+ : readDeployment(request)+ )+ const replaceDeployment = appsApi.replaceNamespacedDeployment.getMockImplementation()!+ appsApi.replaceNamespacedDeployment.mockImplementation(async request => {+ if (request.name !== host.name) return replaceDeployment(request)+ // The replaced pod stays Ready, as a running runtime does.+ live = { ...structuredClone(request.body), status: { readyReplicas: 1 } }+ return live+ })+ const hostDeploymentWrites = () =>+ [+ ...appsApi.createNamespacedDeployment.mock.calls,+ ...appsApi.replaceNamespacedDeployment.mock.calls,+ ...appsApi.patchNamespacedDeployment.mock.calls,+ ].filter(([request]) => (request as { name?: string }).name === host.name)+ return {+ host,+ reconciler,+ appsApi,+ coreApi,+ record: () => record,+ hostDeploymentWrites,+ }+ }++ it('keeps oauth:user-token and does not roll the Deployment when the McpServer probe fails between relists', async () => {+ const { host, reconciler, appsApi, record, hostDeploymentWrites } =+ await oauthGrantedRuntime()+ const oauthResolver = vi.fn(async (): Promise<boolean> => true)+ reconciler.setHostFrontsOAuthServer(oauthResolver)+ const issue = vi.mocked(issueMcpHostRuntimeTokens)+ issue.mockClear()+ for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)+ // The steady state is stable: no runtime mint while the grant is observed.+ expect(issue).not.toHaveBeenCalled()++ oauthResolver.mockImplementation(async () => {+ throw new Error('mcp-server watch retired')+ })+ oauthResolver.mockClear()+ appsApi.createNamespacedDeployment.mockClear()+ appsApi.replaceNamespacedDeployment.mockClear()+ appsApi.patchNamespacedDeployment.mockClear()+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)+ try {+ await reconciler.reconcile(host)++ expect(issue).not.toHaveBeenCalled()+ expect(hostDeploymentWrites()).toHaveLength(0)+ expect(record().metadata!.annotations).toMatchObject({+ 'clerum.io/runtime-token-fronts-oauth-server': 'true',+ 'clerum.io/runtime-token-rollout-required': 'false',+ })+ // Liveness witness: the live probe ran all its attempts and the reconcile+ // deferred with the reason it could not decide.+ expect(oauthResolver).toHaveBeenCalledTimes(3)+ expect(warn).toHaveBeenCalledWith(+ 'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',+ {+ host: host.name,+ namespace: host.namespace,+ observation: 'frontsOAuthServer',+ attempts: 3,+ err: 'mcp-server watch retired',+ }+ )+ expect(reconciler.getStatus(host.name)).toMatchObject({+ deployed: true,+ ready: false,+ message: 'Waiting for authoritative scope observation',+ })+ } finally {+ warn.mockRestore()+ }+ })++ it('lets the Deployment drift guard keep an observed grant when its own McpServer read fails', async () => {+ const { host, reconciler, record } = await oauthGrantedRuntime()+ const oauthResolver = vi.fn(async (): Promise<boolean> => true)+ reconciler.setHostFrontsOAuthServer(oauthResolver)+ const issue = vi.mocked(issueMcpHostRuntimeTokens)+ issue.mockClear()+ for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)+ expect(issue).not.toHaveBeenCalled()+ const steadyStatus = structuredClone(reconciler.getStatus(host.name))++ // Issuance observes the grant; every later read (the drift guard's) fails.+ oauthResolver.mockClear()+ oauthResolver.mockImplementation(async () => {+ if (oauthResolver.mock.calls.length === 1) return true+ throw new Error('mcp-server watch retired')+ })+ await reconciler.reconcile(host)++ // Liveness witness: the drift guard read after issuance and exhausted its retries.+ expect(oauthResolver.mock.calls.length).toBeGreaterThanOrEqual(4)+ expect(issue).not.toHaveBeenCalled()+ expect(reconciler.getStatus(host.name)).toEqual(steadyStatus)+ expect(record().metadata!.annotations).toMatchObject({+ 'clerum.io/runtime-token-fronts-oauth-server': 'true',+ 'clerum.io/runtime-token-rollout-required': 'false',+ })+ })++ it('falls back to a fail-closed mint when the OAuth observation is unavailable inside the renewal window', async () => {+ const { host, reconciler, record } = await oauthGrantedRuntime({+ 'clerum.io/runtime-token-refresh-before': '2000-01-01T00:00:00.000Z',+ })+ const oauthResolver = vi.fn(async (): Promise<boolean> => {+ throw new Error('mcp-server watch retired')+ })+ reconciler.setHostFrontsOAuthServer(oauthResolver)+ const issue = vi.mocked(issueMcpHostRuntimeTokens)+ issue.mockClear()++ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)++ // Liveness witness: the live probe ran all its attempts before the mint.+ expect(oauthResolver).toHaveBeenCalledTimes(3)+ expect(provision).not.toBeNull()+ expect(issue).toHaveBeenCalledOnce()+ expect(issue.mock.calls[0]?.[2]).not.toContain(OAUTH_USER_TOKEN_SCOPE)+ expect(record().metadata!.annotations).toMatchObject({+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',+ 'clerum.io/runtime-token-rollout-required': 'true',+ })+ })+ })+describe('held wake OAuth scope during channel cache loss', () => {function heldZeroReplicaWake(options: {cacheSynced: booleandiff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex 04d697d83..d7eb92e7d 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -2631,15 +2631,24 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {it('fails closed (no oauth:user-token) when the oauth-server probe throws', async () => {vi.mocked(issueMcpHostRuntimeTokens).mockClear()- const { reconciler } = createReconciler()- reconciler.setHostFrontsOAuthServer(async () => {- throw new Error('context read failed')- })+ const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')+ try {+ const { reconciler } = createReconciler()+ reconciler.setHostFrontsOAuthServer(async () => {+ throw new Error('context read failed')+ })- await reconciler.reconcile(makeHost())+ await reconciler.reconcile(makeHost())- expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())- expect(lastIssuedScopes()).not.toContain('oauth:user-token')+ expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())+ expect(lastIssuedScopes()).not.toContain('oauth:user-token')+ // A new Host has no retained grant to preserve, so the unknown answer+ // fails closed at issuance, and says so.+ const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))+ expect(retryWarn?.[1]).toMatchObject({ attempts: 3, err: 'context read failed' })+ } finally {+ warnSpy.mockRestore()+ }})it('toggling oauth-server presence changes the runtime-token scope hash (drift re-issues)', () => {@@ -2671,27 +2680,38 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {})})-describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () => {+describe('HostReconciler.observeFrontsOAuthServer transient-blip retry (R4-M9)', () => {// The oauth-server probe is exercised through the REAL reconciler built by// createReconciler() and the REAL setHostFrontsOAuthServer setter — no- // hand-minted value. We assert on the private method directly (rather than the- // full reconcile) on purpose: the deployment drift guard- // (ensureCurrentRuntimeTokenScope) re-mints the token when the scope hash- // changes mid-reconcile, so a probe that throws once and then returns `true`- // would be MASKED at the public issuance observable — the guard would re-mint- // with the recovered `true` value, hiding the collapse. The boolean returned by- // frontsOAuthServer IS the observable whose flip drives the churn this fixes.- const callFronts = (reconciler: HostReconciler, host: HostCRD): Promise<boolean> =>+ // hand-minted value. We assert on the private observation directly (rather+ // than the full reconcile) on purpose: the deployment drift guard+ // (ensureCurrentRuntimeTokenScope) re-reads the observation mid-reconcile, so+ // a probe that throws once and then returns `true` would be MASKED at the+ // public issuance observable. The observation IS the value whose flip drives+ // the churn this fixes; what an unobserved answer means is decided by its+ // callers and covered at the reconcile level.+ const observeFronts = (+ reconciler: HostReconciler,+ host: HostCRD+ ): Promise<+ | { observed: true; frontsOAuthServer: boolean }+ | { observed: false; attempts: number; error: unknown }+ > =>(reconciler as unknown as {- frontsOAuthServer(host: HostCRD): Promise<boolean>+ observeFrontsOAuthServer(+ host: HostCRD+ ): Promise<+ | { observed: true; frontsOAuthServer: boolean }+ | { observed: false; attempts: number; error: unknown }+ >}- ).frontsOAuthServer(host)+ ).observeFrontsOAuthServer(host)it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {- // Regression case (T3): against the pre-fix head the throw collapses to- // `false` and this assertion fails. With the bounded retry the second- // attempt returns the authoritative `true`.+ // Regression case (T3): without the bounded retry the throw is reported as+ // unobserved and this assertion fails. With the retry the second attempt+ // returns the authoritative `true`.const { reconciler } = createReconciler()let calls = 0reconciler.setHostFrontsOAuthServer(async () => {@@ -2700,7 +2720,10 @@ describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () =>return true})- await expect(callFronts(reconciler, makeHost())).resolves.toBe(true)+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({+ observed: true,+ frontsOAuthServer: true,+ })expect(calls).toBe(2)})@@ -2714,29 +2737,28 @@ describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () =>return false})- await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({+ observed: true,+ frontsOAuthServer: false,+ })expect(calls).toBe(1)})- it('fails closed after exhausting retries: probe always throws → false, N attempts, warn logged', async () => {- const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')- try {- const { reconciler } = createReconciler()- let calls = 0- reconciler.setHostFrontsOAuthServer(async () => {- calls += 1- throw new Error('sustained apiserver outage')- })+ it('reports an unobserved answer after exhausting retries instead of collapsing it to false', async () => {+ const { reconciler } = createReconciler()+ let calls = 0+ const outage = new Error('sustained apiserver outage')+ reconciler.setHostFrontsOAuthServer(async () => {+ calls += 1+ throw outage+ })- await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)- // Retried up to the bounded ceiling (fix uses 3 total attempts); pre-fix- // head would call the probe exactly once.- expect(calls).toBe(3)- const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))- expect(retryWarn).toBeDefined()- expect(retryWarn?.[1]).toMatchObject({ attempts: 3 })- } finally {- warnSpy.mockRestore()- }+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({+ observed: false,+ attempts: 3,+ error: outage,+ })+ // Retried up to the bounded ceiling (3 total attempts).+ expect(calls).toBe(3)})})