Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): defer instead of downgrading oauth:user-token when the McpServer read is unobservable

Alfredo Lopez2327789ed5e6
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 3923e9bb0..4792a9463 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index a3198cbcc..4e4821e70 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.370",
+ "version": "0.1.371",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 6c3c316da..4ed7020ac 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -708,46 +708,20 @@ export class HostReconciler {
/**
* Whether this Host fronts an enabled `auth.type: oauth` mcp-server, gating
- * the derive-only `oauth:user-token` runtime scope. Fails CLOSED (false) when
- * the cross-CRD read throws: an uncertain read must never grant the scope.
+ * the derive-only `oauth:user-token` runtime scope. A read that failed after
+ * its retries is reported as unobserved instead of collapsing to `false`:
+ * "scope unknown" is not "scope revoked". Callers decide what an unknown
+ * answer means — the issuance path fails closed only where that cannot strip
+ * a grant the running pod holds, and the deployment drift guard
+ * (`currentScopeMatchesProvision`) never forces a re-mint on it. An uncertain
+ * read never grants the scope.
*
* Each call is an INDEPENDENT live point-in-time read (mirrors
* `hasChannelIngress`); the value is deliberately not cached across a
- * reconcile. Callers that need one consistent value within a unit of work
- * resolve it once and thread the same bool (see the issuance path, which
- * threads it into the mint-scope derive and the drift hash so they never
- * diverge); the deployment drift guard re-reads it on purpose to catch a
- * scope change mid-reconcile.
- */
- private async frontsOAuthServer(host: HostCRD): Promise<boolean> {
- const observation = await this.observeFrontsOAuthServer(host)
- if (observation.observed) return observation.frontsOAuthServer
- // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every
- // attempt throws) still fails CLOSED here and can flip the scope off. That is
- // correct — a sustained outage is a genuine "scope unknown" situation, not a
- // blip — the retry only smooths over transient blips, never a real outage.
- log.warn(
- 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
- {
- host: host.name,
- namespace: host.namespace,
- contextRef: host.spec.contextRef,
- attempts: observation.attempts,
- err:
- observation.error instanceof Error
- ? observation.error.message
- : String(observation.error),
- }
- )
- return false
- }
-
- /**
- * The live McpServer-backed OAuth observation behind `frontsOAuthServer`,
- * reporting a read that failed after its retries instead of collapsing it to
- * `false`. The CommunicationChannel-loss issuance path needs that distinction:
- * it has no authoritative channel inventory, so it must not replace an
- * unknown OAuth answer with a retained one that could re-grant a revoked scope.
+ * reconcile. The issuance path resolves it once and threads the same bool
+ * into the mint-scope derive and the drift hash so they never diverge; the
+ * deployment drift guard re-reads it on purpose to catch a scope change
+ * mid-reconcile.
*/
private async observeFrontsOAuthServer(
host: HostCRD
@@ -1831,6 +1805,28 @@ export class HostReconciler {
)
}
+ /**
+ * Deployment drift guard: does the provisioned credential still match the
+ * Host's current scope contract? An OAuth read that failed after its retries
+ * is "scope unknown", so it matches a provision minted under either OAuth
+ * answer instead of collapsing to `false` and forcing a re-mint that would
+ * strip oauth:user-token and roll the Deployment. The issuance path owns the
+ * decision to narrow the grant (see `ensureMcpHostRuntimeTokenSecret`).
+ */
+ private currentScopeMatchesProvision(
+ host: HostCRD,
+ provision: RuntimeTokenProvision,
+ live: Awaited<ReturnType<HostReconciler['observeFrontsOAuthServer']>>
+ ): boolean {
+ if (live.observed) {
+ return provision.scopeHash === this.runtimeScopeHashFor(host, live.frontsOAuthServer)
+ }
+ return (
+ provision.scopeHash === this.runtimeScopeHashFor(host, true) ||
+ provision.scopeHash === this.runtimeScopeHashFor(host, false)
+ )
+ }
+
private async refreshCodexSnapshot(): Promise<void> {
try {
const cm = await this.coreApi.readNamespacedConfigMap({
@@ -2050,6 +2046,78 @@ export class HostReconciler {
)
return null
}
+ const projection = this.projectCodexForHost(host)
+ const grokProjection = this.projectGrokForHost(host)
+ const decideRefresh = (
+ frontsOAuth: boolean
+ ): {
+ refresh: boolean
+ rolloutRequired: boolean
+ reason: string
+ refreshTokenExpMs?: number
+ } => {
+ let decided: {
+ refresh: boolean
+ rolloutRequired: boolean
+ reason: string
+ refreshTokenExpMs?: number
+ } = existing
+ ? HostReconciler.runtimeTokenRefreshDecision(
+ host,
+ existing,
+ nowMs,
+ hasChannelIngress,
+ frontsOAuth,
+ projection,
+ grokProjection
+ )
+ : { refresh: true, rolloutRequired: false, reason: 'missing_secret' }
+ if (
+ existing &&
+ HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)
+ ) {
+ // A manual restart after the runtime Secret was issued starts pods with
+ // env-var bootstrap tokens from that older Secret. Force a re-issue and
+ // rollout so every restarted pod observes a fresh, unconsumed token pair.
+ decided = {
+ refresh: true,
+ rolloutRequired: true,
+ reason: 'deployment_restarted_after_secret_issued',
+ }
+ }
+
+ // Revoked-on-wake guard: the mcp-host runtime refresh token is
+ // single-use-rotating (control-api revokes the prior JTI on every
+ // refresh). A pod that is about to boot -- wake (scale 0->1), missing
+ // Deployment, or a not-Ready (re)starting pod -- reads its bootstrap
+ // refresh token from THIS Secret. If the pre-suspend pod already
+ // rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"
+ // hands the booting pod a dead token and readiness 503s forever. When
+ // a new pod will consume the Secret, force a fresh mint + rollout even
+ // though the decoded exp is far out. Reuse stays correct only for a
+ // running, Ready pod that never re-reads the Secret (churn fix).
+ if (
+ existingRevision &&
+ decided.refresh === false &&
+ HostReconciler.deploymentNeedsFreshBootstrap(
+ deployment,
+ existingRevision,
+ existing,
+ options
+ )
+ ) {
+ decided = {
+ refresh: true,
+ rolloutRequired: true,
+ reason: 'fresh_mint_for_booting_pod',
+ ...(decided.refreshTokenExpMs !== undefined
+ ? { refreshTokenExpMs: decided.refreshTokenExpMs }
+ : {}),
+ }
+ }
+ return decided
+ }
+
// Resolve ONCE per issuance and thread the SAME frontsOAuthServer bool and
// codex projection into the mint-scope derive, the refresh decision, the
// scope hash and the stored annotation, so the minted token and the drift
@@ -2059,37 +2127,59 @@ export class HostReconciler {
// on the CommunicationChannel cache, so it stays live during channel
// cache loss: an admin who removed the OAuth mcp-server while the channel
// watch was down must not have `oauth:user-token` re-granted from the
- // retained annotation on a held wake. The retained value is used only
- // when the live read failed after its retries and it cannot widen the
- // grant (`false`); otherwise the mint is skipped (fail closed).
+ // retained annotation on a held wake.
+ //
+ // A live read that failed after its retries is "scope unknown", not
+ // "scope revoked". It resolves to `false` (fail closed) only when that
+ // cannot narrow a grant the running pod holds: the retained value is
+ // `false`, or there is no trusted retained value while channel authority
+ // is synced. A retained `true` under synced authority defers the decision
+ // while the credential needs no renewal, so an McpServer relist does not
+ // strip oauth:user-token and roll the Deployment; once the credential must
+ // be renewed, the renewal mints without the unobserved scope. Under channel
+ // cache loss the mint is skipped instead.
+ const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)
let observedFrontsOAuth: boolean
- if (cacheSynced) {
- observedFrontsOAuth = await this.frontsOAuthServer(host)
+ if (liveFrontsOAuth.observed) {
+ observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer
} else {
- const liveFrontsOAuth = await this.observeFrontsOAuthServer(host)
- if (liveFrontsOAuth.observed) {
- observedFrontsOAuth = liveFrontsOAuth.frontsOAuthServer
- } else if (retainedFrontsOAuth === false) {
+ const observationFailure = {
+ host: host.name,
+ namespace: host.namespace,
+ observation: 'frontsOAuthServer',
+ attempts: liveFrontsOAuth.attempts,
+ err:
+ liveFrontsOAuth.error instanceof Error
+ ? liveFrontsOAuth.error.message
+ : String(liveFrontsOAuth.error),
+ }
+ if (retainedFrontsOAuth === false || (cacheSynced && retainedFrontsOAuth !== true)) {
+ log.warn(
+ 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
+ observationFailure
+ )
+ observedFrontsOAuth = false
+ } else if (cacheSynced) {
+ if (!decideRefresh(true).refresh) {
+ log.warn(
+ 'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',
+ observationFailure
+ )
+ return null
+ }
+ log.warn(
+ 'renewing runtime token without oauth:user-token: OAuth observation unavailable when the credential must be renewed',
+ observationFailure
+ )
observedFrontsOAuth = false
} else {
log.warn(
'skipping runtime token mint during channel cache loss without an authoritative OAuth observation',
- {
- host: host.name,
- namespace: host.namespace,
- observation: 'frontsOAuthServer',
- attempts: liveFrontsOAuth.attempts,
- err:
- liveFrontsOAuth.error instanceof Error
- ? liveFrontsOAuth.error.message
- : String(liveFrontsOAuth.error),
- }
+ observationFailure
)
return null
}
}
- const projection = this.projectCodexForHost(host)
- const grokProjection = this.projectGrokForHost(host)
const scopeHash = HostReconciler.runtimeTokenScopeHash(
host,
hasChannelIngress,
@@ -2097,65 +2187,7 @@ export class HostReconciler {
projection,
grokProjection
)
- let decision: {
- refresh: boolean
- rolloutRequired: boolean
- reason: string
- refreshTokenExpMs?: number
- } = existing
- ? HostReconciler.runtimeTokenRefreshDecision(
- host,
- existing,
- nowMs,
- hasChannelIngress,
- observedFrontsOAuth,
- projection,
- grokProjection
- )
- : { refresh: true, rolloutRequired: false, reason: 'missing_secret' }
- if (
- existing &&
- HostReconciler.deploymentRestartedAfterRuntimeSecretIssued(existing, deployment)
- ) {
- // A manual restart after the runtime Secret was issued starts pods with
- // env-var bootstrap tokens from that older Secret. Force a re-issue and
- // rollout so every restarted pod observes a fresh, unconsumed token pair.
- decision = {
- refresh: true,
- rolloutRequired: true,
- reason: 'deployment_restarted_after_secret_issued',
- }
- }
-
- // Revoked-on-wake guard: the mcp-host runtime refresh token is
- // single-use-rotating (control-api revokes the prior JTI on every
- // refresh). A pod that is about to boot -- wake (scale 0->1), missing
- // Deployment, or a not-Ready (re)starting pod -- reads its bootstrap
- // refresh token from THIS Secret. If the pre-suspend pod already
- // rotated, the copy in the Secret is REVOKED, so an exp-based "reuse"
- // hands the booting pod a dead token and readiness 503s forever. When
- // a new pod will consume the Secret, force a fresh mint + rollout even
- // though the decoded exp is far out. Reuse stays correct only for a
- // running, Ready pod that never re-reads the Secret (churn fix).
- if (
- existingRevision &&
- decision.refresh === false &&
- HostReconciler.deploymentNeedsFreshBootstrap(
- deployment,
- existingRevision,
- existing,
- options
- )
- ) {
- decision = {
- refresh: true,
- rolloutRequired: true,
- reason: 'fresh_mint_for_booting_pod',
- ...(decision.refreshTokenExpMs !== undefined
- ? { refreshTokenExpMs: decision.refreshTokenExpMs }
- : {}),
- }
- }
+ const decision = decideRefresh(observedFrontsOAuth)
if (existing && existingRevision && !decision.refresh) {
const deploymentRevision = HostReconciler.deploymentRuntimeTokenRevision(deployment)
@@ -5411,8 +5443,10 @@ export class HostReconciler {
const ensureCurrentRuntimeTokenScope = async (): Promise<void> => {
for (let attempt = 1; attempt <= 3; attempt++) {
revalidateHostMutationBoundary()
- const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))
- if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) return
+ const currentOAuth = await this.observeFrontsOAuthServer(host)
+ if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
+ return
+ }
runtimeTokenProvision = await this.provisionRuntimeTokenRevision(host, {
forceFreshForWake: false,
@@ -5425,11 +5459,16 @@ export class HostReconciler {
throw error
}
revalidateHostMutationBoundary()
- const postProvisionScopeHash = this.runtimeScopeHashFor(
- host,
- await this.frontsOAuthServer(host)
- )
- if (requireRuntimeTokenProvision().scopeHash === postProvisionScopeHash) return
+ const postProvisionOAuth = await this.observeFrontsOAuthServer(host)
+ if (
+ this.currentScopeMatchesProvision(
+ host,
+ requireRuntimeTokenProvision(),
+ postProvisionOAuth
+ )
+ ) {
+ return
+ }
log.warn('CommunicationChannel scope contract changed during token provisioning', {
host: host.name,
@@ -5455,7 +5494,7 @@ export class HostReconciler {
}
await ensureCurrentRuntimeTokenScope()
const effective = await resolveDeploymentLifecycle()
- const currentScopeHash = this.runtimeScopeHashFor(host, await this.frontsOAuthServer(host))
+ const currentOAuth = await this.observeFrontsOAuthServer(host)
// The OAuth lookup can await I/O after the last lifecycle observation.
// Retry assessment rather than committing a now-obsolete mode.
if (
@@ -5464,7 +5503,7 @@ export class HostReconciler {
) {
continue
}
- if (requireRuntimeTokenProvision().scopeHash === currentScopeHash) {
+ if (this.currentScopeMatchesProvision(host, requireRuntimeTokenProvision(), currentOAuth)) {
revalidateHostMutationBoundary()
return {
lifecycle: effective,
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 8a600283a..12abf3866 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -1735,12 +1735,17 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
vi.spyOn(reconciler as any, 'provisionRuntimeTokenRevision').mockResolvedValue(
runtimeTokenProvision(host)
)
- vi.spyOn(reconciler as any, 'frontsOAuthServer').mockImplementation(async () => {
- if (++lookup === 2) synced = false
- return false
+ // The final lookup loses channel authority and its McpServer read fails too:
+ // neither an unobservable OAuth answer nor the lost authority may commit
+ // the template.
+ reconciler.setHostFrontsOAuthServer(async () => {
+ if (++lookup === 1) return false
+ synced = false
+ throw new Error('mcp-server watch retired')
})
await reconciler.reconcile(host)
- expect(lookup).toBe(2)
+ // One observed lookup, then the final lookup's three attempts.
+ expect(lookup).toBe(4)
expect(live().spec!.template).toEqual(applied.spec!.template)
expect(rejectedCondition(lifecycleStatusWrites(customApi).at(-1)!).reason).toBe(
'CommunicationChannelCacheUnsynced'
@@ -1808,7 +1813,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
expect(reconciler.getStatus(host.name)).toMatchObject({ deployed: false, ready: false })
})
- it('does not let a retained OAuth observation override a failed live probe', async () => {
+ it('does not let a failed live probe narrow a retained OAuth grant before its renewal window', async () => {
const host = makeStatelessHost()
const { reconciler, coreApi } = createReconciler({
isCommunicationChannelCacheSynced: () => true,
@@ -1816,21 +1821,21 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
coreApi.readNamespacedSecret.mockResolvedValue(
await mintedRuntimeCredentialRecord(host, { frontsOAuthServer: true })
)
- reconciler.setHostFrontsOAuthServer(async () => {
+ const oauthResolver = vi.fn(async (): Promise<boolean> => {
throw new Error('oauth lookup unavailable')
})
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
- expect(provision).not.toBeNull()
- expect(vi.mocked(issueMcpHostRuntimeTokens).mock.calls.at(-1)?.[2]).not.toContain(
- OAUTH_USER_TOKEN_SCOPE
- )
- const write = coreApi.replaceNamespacedSecret.mock.calls.at(-1)?.[0].body as k8s.V1Secret
- expect(write.metadata?.annotations?.['clerum.io/runtime-token-fronts-oauth-server']).toBe(
- 'false'
- )
+ // Liveness witness: the record was read and the live probe ran all its attempts.
+ expect(coreApi.readNamespacedSecret).toHaveBeenCalled()
+ expect(oauthResolver).toHaveBeenCalledTimes(3)
+ expect(provision).toBeNull()
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
+ expect(coreApi.replaceNamespacedSecret).not.toHaveBeenCalled()
+ expect(coreApi.createNamespacedSecret).not.toHaveBeenCalled()
})
it('uses the retained channel observation but a live OAuth observation while channel authority is unavailable', async () => {
@@ -1865,6 +1870,166 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
})
})
+ describe('McpServer OAuth observation unavailable while channel authority is synced', () => {
+ /**
+ * A Host whose Ready pod runs a consumed credential that grants
+ * oauth:user-token, over stateful Secret and Deployment mocks.
+ */
+ async function oauthGrantedRuntime(annotations: Record<string, string> = {}) {
+ const host = makeHost()
+ const { reconciler, appsApi, coreApi } = createReconciler()
+ let record = await mintedRuntimeCredentialRecord(host, {
+ frontsOAuthServer: true,
+ annotations,
+ })
+ const secretName = record.metadata!.name!
+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
+ coreApi.readNamespacedSecret.mockImplementation(request =>
+ request.name === secretName ? Promise.resolve(structuredClone(record)) : readSecret(request)
+ )
+ coreApi.replaceNamespacedSecret.mockImplementation(async request => {
+ if (request.name === secretName) record = structuredClone(request.body as k8s.V1Secret)
+ return request.body
+ })
+ let live = trustedRuntimeDeployment(reconciler, host)
+ live.spec!.template!.metadata!.annotations = {
+ ...live.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision':
+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
+ }
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ request.name === host.name
+ ? Promise.resolve(structuredClone(live))
+ : readDeployment(request)
+ )
+ const replaceDeployment = appsApi.replaceNamespacedDeployment.getMockImplementation()!
+ appsApi.replaceNamespacedDeployment.mockImplementation(async request => {
+ if (request.name !== host.name) return replaceDeployment(request)
+ // The replaced pod stays Ready, as a running runtime does.
+ live = { ...structuredClone(request.body), status: { readyReplicas: 1 } }
+ return live
+ })
+ const hostDeploymentWrites = () =>
+ [
+ ...appsApi.createNamespacedDeployment.mock.calls,
+ ...appsApi.replaceNamespacedDeployment.mock.calls,
+ ...appsApi.patchNamespacedDeployment.mock.calls,
+ ].filter(([request]) => (request as { name?: string }).name === host.name)
+ return {
+ host,
+ reconciler,
+ appsApi,
+ coreApi,
+ record: () => record,
+ hostDeploymentWrites,
+ }
+ }
+
+ it('keeps oauth:user-token and does not roll the Deployment when the McpServer probe fails between relists', async () => {
+ const { host, reconciler, appsApi, record, hostDeploymentWrites } =
+ await oauthGrantedRuntime()
+ const oauthResolver = vi.fn(async (): Promise<boolean> => true)
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
+ const issue = vi.mocked(issueMcpHostRuntimeTokens)
+ issue.mockClear()
+ for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)
+ // The steady state is stable: no runtime mint while the grant is observed.
+ expect(issue).not.toHaveBeenCalled()
+
+ oauthResolver.mockImplementation(async () => {
+ throw new Error('mcp-server watch retired')
+ })
+ oauthResolver.mockClear()
+ appsApi.createNamespacedDeployment.mockClear()
+ appsApi.replaceNamespacedDeployment.mockClear()
+ appsApi.patchNamespacedDeployment.mockClear()
+ const warn = vi.spyOn(HostContextLogger.prototype, 'warn').mockImplementation(() => undefined)
+ try {
+ await reconciler.reconcile(host)
+
+ expect(issue).not.toHaveBeenCalled()
+ expect(hostDeploymentWrites()).toHaveLength(0)
+ expect(record().metadata!.annotations).toMatchObject({
+ 'clerum.io/runtime-token-fronts-oauth-server': 'true',
+ 'clerum.io/runtime-token-rollout-required': 'false',
+ })
+ // Liveness witness: the live probe ran all its attempts and the reconcile
+ // deferred with the reason it could not decide.
+ expect(oauthResolver).toHaveBeenCalledTimes(3)
+ expect(warn).toHaveBeenCalledWith(
+ 'deferring runtime token decision: OAuth observation unavailable and retained scope grants oauth:user-token',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ observation: 'frontsOAuthServer',
+ attempts: 3,
+ err: 'mcp-server watch retired',
+ }
+ )
+ expect(reconciler.getStatus(host.name)).toMatchObject({
+ deployed: true,
+ ready: false,
+ message: 'Waiting for authoritative scope observation',
+ })
+ } finally {
+ warn.mockRestore()
+ }
+ })
+
+ it('lets the Deployment drift guard keep an observed grant when its own McpServer read fails', async () => {
+ const { host, reconciler, record } = await oauthGrantedRuntime()
+ const oauthResolver = vi.fn(async (): Promise<boolean> => true)
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
+ const issue = vi.mocked(issueMcpHostRuntimeTokens)
+ issue.mockClear()
+ for (let pass = 1; pass <= 3; pass++) await reconciler.reconcile(host)
+ expect(issue).not.toHaveBeenCalled()
+ const steadyStatus = structuredClone(reconciler.getStatus(host.name))
+
+ // Issuance observes the grant; every later read (the drift guard's) fails.
+ oauthResolver.mockClear()
+ oauthResolver.mockImplementation(async () => {
+ if (oauthResolver.mock.calls.length === 1) return true
+ throw new Error('mcp-server watch retired')
+ })
+ await reconciler.reconcile(host)
+
+ // Liveness witness: the drift guard read after issuance and exhausted its retries.
+ expect(oauthResolver.mock.calls.length).toBeGreaterThanOrEqual(4)
+ expect(issue).not.toHaveBeenCalled()
+ expect(reconciler.getStatus(host.name)).toEqual(steadyStatus)
+ expect(record().metadata!.annotations).toMatchObject({
+ 'clerum.io/runtime-token-fronts-oauth-server': 'true',
+ 'clerum.io/runtime-token-rollout-required': 'false',
+ })
+ })
+
+ it('falls back to a fail-closed mint when the OAuth observation is unavailable inside the renewal window', async () => {
+ const { host, reconciler, record } = await oauthGrantedRuntime({
+ 'clerum.io/runtime-token-refresh-before': '2000-01-01T00:00:00.000Z',
+ })
+ const oauthResolver = vi.fn(async (): Promise<boolean> => {
+ throw new Error('mcp-server watch retired')
+ })
+ reconciler.setHostFrontsOAuthServer(oauthResolver)
+ const issue = vi.mocked(issueMcpHostRuntimeTokens)
+ issue.mockClear()
+
+ const provision = await (reconciler as any).ensureMcpHostRuntimeTokenSecret(host)
+
+ // Liveness witness: the live probe ran all its attempts before the mint.
+ expect(oauthResolver).toHaveBeenCalledTimes(3)
+ expect(provision).not.toBeNull()
+ expect(issue).toHaveBeenCalledOnce()
+ expect(issue.mock.calls[0]?.[2]).not.toContain(OAUTH_USER_TOKEN_SCOPE)
+ expect(record().metadata!.annotations).toMatchObject({
+ 'clerum.io/runtime-token-fronts-oauth-server': 'false',
+ 'clerum.io/runtime-token-rollout-required': 'true',
+ })
+ })
+ })
+
describe('held wake OAuth scope during channel cache loss', () => {
function heldZeroReplicaWake(options: {
cacheSynced: boolean
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index 04d697d83..d7eb92e7d 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -2631,15 +2631,24 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {
it('fails closed (no oauth:user-token) when the oauth-server probe throws', async () => {
vi.mocked(issueMcpHostRuntimeTokens).mockClear()
- const { reconciler } = createReconciler()
- reconciler.setHostFrontsOAuthServer(async () => {
- throw new Error('context read failed')
- })
+ const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')
+ try {
+ const { reconciler } = createReconciler()
+ reconciler.setHostFrontsOAuthServer(async () => {
+ throw new Error('context read failed')
+ })
- await reconciler.reconcile(makeHost())
+ await reconciler.reconcile(makeHost())
- expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
- expect(lastIssuedScopes()).not.toContain('oauth:user-token')
+ expect(lastIssuedScopes()).toEqual([...DEFAULT_FIRST_PARTY_WORKFLOW_CONTROL_SCOPES].sort())
+ expect(lastIssuedScopes()).not.toContain('oauth:user-token')
+ // A new Host has no retained grant to preserve, so the unknown answer
+ // fails closed at issuance, and says so.
+ const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))
+ expect(retryWarn?.[1]).toMatchObject({ attempts: 3, err: 'context read failed' })
+ } finally {
+ warnSpy.mockRestore()
+ }
})
it('toggling oauth-server presence changes the runtime-token scope hash (drift re-issues)', () => {
@@ -2671,27 +2680,38 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {
})
})
-describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () => {
+describe('HostReconciler.observeFrontsOAuthServer transient-blip retry (R4-M9)', () => {
// The oauth-server probe is exercised through the REAL reconciler built by
// createReconciler() and the REAL setHostFrontsOAuthServer setter — no
- // hand-minted value. We assert on the private method directly (rather than the
- // full reconcile) on purpose: the deployment drift guard
- // (ensureCurrentRuntimeTokenScope) re-mints the token when the scope hash
- // changes mid-reconcile, so a probe that throws once and then returns `true`
- // would be MASKED at the public issuance observable — the guard would re-mint
- // with the recovered `true` value, hiding the collapse. The boolean returned by
- // frontsOAuthServer IS the observable whose flip drives the churn this fixes.
- const callFronts = (reconciler: HostReconciler, host: HostCRD): Promise<boolean> =>
+ // hand-minted value. We assert on the private observation directly (rather
+ // than the full reconcile) on purpose: the deployment drift guard
+ // (ensureCurrentRuntimeTokenScope) re-reads the observation mid-reconcile, so
+ // a probe that throws once and then returns `true` would be MASKED at the
+ // public issuance observable. The observation IS the value whose flip drives
+ // the churn this fixes; what an unobserved answer means is decided by its
+ // callers and covered at the reconcile level.
+ const observeFronts = (
+ reconciler: HostReconciler,
+ host: HostCRD
+ ): Promise<
+ | { observed: true; frontsOAuthServer: boolean }
+ | { observed: false; attempts: number; error: unknown }
+ > =>
(
reconciler as unknown as {
- frontsOAuthServer(host: HostCRD): Promise<boolean>
+ observeFrontsOAuthServer(
+ host: HostCRD
+ ): Promise<
+ | { observed: true; frontsOAuthServer: boolean }
+ | { observed: false; attempts: number; error: unknown }
+ >
}
- ).frontsOAuthServer(host)
+ ).observeFrontsOAuthServer(host)
it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {
- // Regression case (T3): against the pre-fix head the throw collapses to
- // `false` and this assertion fails. With the bounded retry the second
- // attempt returns the authoritative `true`.
+ // Regression case (T3): without the bounded retry the throw is reported as
+ // unobserved and this assertion fails. With the retry the second attempt
+ // returns the authoritative `true`.
const { reconciler } = createReconciler()
let calls = 0
reconciler.setHostFrontsOAuthServer(async () => {
@@ -2700,7 +2720,10 @@ describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () =>
return true
})
- await expect(callFronts(reconciler, makeHost())).resolves.toBe(true)
+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
+ observed: true,
+ frontsOAuthServer: true,
+ })
expect(calls).toBe(2)
})
@@ -2714,29 +2737,28 @@ describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () =>
return false
})
- await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)
+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
+ observed: true,
+ frontsOAuthServer: false,
+ })
expect(calls).toBe(1)
})
- it('fails closed after exhausting retries: probe always throws → false, N attempts, warn logged', async () => {
- const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')
- try {
- const { reconciler } = createReconciler()
- let calls = 0
- reconciler.setHostFrontsOAuthServer(async () => {
- calls += 1
- throw new Error('sustained apiserver outage')
- })
+ it('reports an unobserved answer after exhausting retries instead of collapsing it to false', async () => {
+ const { reconciler } = createReconciler()
+ let calls = 0
+ const outage = new Error('sustained apiserver outage')
+ reconciler.setHostFrontsOAuthServer(async () => {
+ calls += 1
+ throw outage
+ })
- await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)
- // Retried up to the bounded ceiling (fix uses 3 total attempts); pre-fix
- // head would call the probe exactly once.
- expect(calls).toBe(3)
- const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))
- expect(retryWarn).toBeDefined()
- expect(retryWarn?.[1]).toMatchObject({ attempts: 3 })
- } finally {
- warnSpy.mockRestore()
- }
+ await expect(observeFronts(reconciler, makeHost())).resolves.toEqual({
+ observed: false,
+ attempts: 3,
+ error: outage,
+ })
+ // Retried up to the bounded ceiling (3 total attempts).
+ expect(calls).toBe(3)
})
})