Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(host-context-controller): retry frontsOAuthServer on transient blips (R4-M9)

Diego Zacarias687eeceaf397
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index fd8dcd343..ac6fc1214 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.251",
+ "version": "0.1.252",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.251",
+ "version": "0.1.252",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 8e8a38126..4729d0eb1 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.251",
+ "version": "0.1.252",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index a62ebebd7..d68bccedb 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -675,20 +675,46 @@ export class HostReconciler {
* scope change mid-reconcile.
*/
private async frontsOAuthServer(host: HostCRD): Promise<boolean> {
- try {
- return await this.hostFrontsOAuthServerFn(host)
- } catch (err) {
- log.warn(
- 'failed to resolve whether Host fronts an oauth mcp-server; failing closed (no oauth:user-token scope)',
- {
- host: host.name,
- namespace: host.namespace,
- contextRef: host.spec.contextRef,
- err: err instanceof Error ? err.message : String(err),
+ // Bounded retry that absorbs a TRANSIENT apiserver blip without touching the
+ // authoritative-`false` path. Contract of the underlying resolver: a real
+ // scope change (the server genuinely stopped fronting an oauth mcp-server)
+ // RETURNS `false` without throwing; only a transient read failure (apiserver
+ // blip) THROWS. So we retry ONLY on throw — any RETURNED value (true or
+ // false) is authoritative and is returned immediately, so a healthy call
+ // resolves on the first attempt with zero added overhead and an authoritative
+ // `false` is never re-tried. Before this retry, a transient throw collapsed
+ // to `false` exactly like an authoritative `false`, dropping the
+ // oauth:user-token scope → runtime-token re-mint + Deployment rollout, then a
+ // flip back on recovery → churn, once per OAuth Host per reconcile.
+ const MAX_ATTEMPTS = 3
+ const BACKOFF_BASE_MS = 50
+ let lastErr: unknown
+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+ try {
+ return await this.hostFrontsOAuthServerFn(host)
+ } catch (err) {
+ lastErr = err
+ if (attempt < MAX_ATTEMPTS) {
+ // Small linear backoff (50ms, 100ms) to let a transient blip clear.
+ await new Promise(resolve => setTimeout(resolve, BACKOFF_BASE_MS * attempt))
}
- )
- return false
+ }
}
+ // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every
+ // attempt throws) still fails CLOSED here and can flip the scope off. That is
+ // correct — a sustained outage is a genuine "scope unknown" situation, not a
+ // blip — the retry only smooths over transient blips, never a real outage.
+ log.warn(
+ 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',
+ {
+ host: host.name,
+ namespace: host.namespace,
+ contextRef: host.spec.contextRef,
+ attempts: MAX_ATTEMPTS,
+ err: lastErr instanceof Error ? lastErr.message : String(lastErr),
+ }
+ )
+ return false
}
/**
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index 59a918bdf..88bf18133 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -10,6 +10,7 @@ import {
resolveWorkflowControlScopes,
} from '../src/hostReconciler'
import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'
+import { HostContextLogger } from '../src/logger'
import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'
import { HostCRD, type HostWorkflowControlScope } from '../src/types'
import {
@@ -2047,3 +2048,73 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {
expect(hashInput).toContain('oauth:user-token')
})
})
+
+describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () => {
+ // The oauth-server probe is exercised through the REAL reconciler built by
+ // createReconciler() and the REAL setHostFrontsOAuthServer setter — no
+ // hand-minted value. We assert on the private method directly (rather than the
+ // full reconcile) on purpose: the deployment drift guard
+ // (ensureCurrentRuntimeTokenScope) re-mints the token when the scope hash
+ // changes mid-reconcile, so a probe that throws once and then returns `true`
+ // would be MASKED at the public issuance observable — the guard would re-mint
+ // with the recovered `true` value, hiding the collapse. The boolean returned by
+ // frontsOAuthServer IS the observable whose flip drives the churn this fixes.
+ const callFronts = (reconciler: HostReconciler, host: HostCRD): Promise<boolean> =>
+ (
+ reconciler as unknown as {
+ frontsOAuthServer(host: HostCRD): Promise<boolean>
+ }
+ ).frontsOAuthServer(host)
+
+ it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {
+ // Regression case (T3): against the pre-fix head the throw collapses to
+ // `false` and this assertion fails. With the bounded retry the second
+ // attempt returns the authoritative `true`.
+ const { reconciler } = createReconciler()
+ let calls = 0
+ reconciler.setHostFrontsOAuthServer(async () => {
+ calls += 1
+ if (calls === 1) throw new Error('transient apiserver read blip')
+ return true
+ })
+
+ await expect(callFronts(reconciler, makeHost())).resolves.toBe(true)
+ expect(calls).toBe(2)
+ })
+
+ it('does NOT retry an authoritative false: probe returns false without throwing → false, called once', async () => {
+ // Control: a real scope change RETURNS false; the retry must not fire, or a
+ // genuine "no longer oauth" would be smeared into a stale `true`.
+ const { reconciler } = createReconciler()
+ let calls = 0
+ reconciler.setHostFrontsOAuthServer(async () => {
+ calls += 1
+ return false
+ })
+
+ await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)
+ expect(calls).toBe(1)
+ })
+
+ it('fails closed after exhausting retries: probe always throws → false, N attempts, warn logged', async () => {
+ const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')
+ try {
+ const { reconciler } = createReconciler()
+ let calls = 0
+ reconciler.setHostFrontsOAuthServer(async () => {
+ calls += 1
+ throw new Error('sustained apiserver outage')
+ })
+
+ await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)
+ // Retried up to the bounded ceiling (fix uses 3 total attempts); pre-fix
+ // head would call the probe exactly once.
+ expect(calls).toBe(3)
+ const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))
+ expect(retryWarn).toBeDefined()
+ expect(retryWarn?.[1]).toMatchObject({ attempts: 3 })
+ } finally {
+ warnSpy.mockRestore()
+ }
+ })
+})