Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(host-context-controller): retry frontsOAuthServer on transient blips (R4-M9)
Diego Zacarias687eeceaf397
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex fd8dcd343..ac6fc1214 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.251",+ "version": "0.1.252","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.251",+ "version": "0.1.252","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 8e8a38126..4729d0eb1 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.251",+ "version": "0.1.252","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex a62ebebd7..d68bccedb 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -675,20 +675,46 @@ export class HostReconciler {* scope change mid-reconcile.*/private async frontsOAuthServer(host: HostCRD): Promise<boolean> {- try {- return await this.hostFrontsOAuthServerFn(host)- } catch (err) {- log.warn(- 'failed to resolve whether Host fronts an oauth mcp-server; failing closed (no oauth:user-token scope)',- {- host: host.name,- namespace: host.namespace,- contextRef: host.spec.contextRef,- err: err instanceof Error ? err.message : String(err),+ // Bounded retry that absorbs a TRANSIENT apiserver blip without touching the+ // authoritative-`false` path. Contract of the underlying resolver: a real+ // scope change (the server genuinely stopped fronting an oauth mcp-server)+ // RETURNS `false` without throwing; only a transient read failure (apiserver+ // blip) THROWS. So we retry ONLY on throw — any RETURNED value (true or+ // false) is authoritative and is returned immediately, so a healthy call+ // resolves on the first attempt with zero added overhead and an authoritative+ // `false` is never re-tried. Before this retry, a transient throw collapsed+ // to `false` exactly like an authoritative `false`, dropping the+ // oauth:user-token scope → runtime-token re-mint + Deployment rollout, then a+ // flip back on recovery → churn, once per OAuth Host per reconcile.+ const MAX_ATTEMPTS = 3+ const BACKOFF_BASE_MS = 50+ let lastErr: unknown+ for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {+ try {+ return await this.hostFrontsOAuthServerFn(host)+ } catch (err) {+ lastErr = err+ if (attempt < MAX_ATTEMPTS) {+ // Small linear backoff (50ms, 100ms) to let a transient blip clear.+ await new Promise(resolve => setTimeout(resolve, BACKOFF_BASE_MS * attempt))}- )- return false+ }}+ // All attempts threw. Residual honesty: a SUSTAINED apiserver outage (every+ // attempt throws) still fails CLOSED here and can flip the scope off. That is+ // correct — a sustained outage is a genuine "scope unknown" situation, not a+ // blip — the retry only smooths over transient blips, never a real outage.+ log.warn(+ 'failed to resolve whether Host fronts an oauth mcp-server after retries; failing closed (no oauth:user-token scope)',+ {+ host: host.name,+ namespace: host.namespace,+ contextRef: host.spec.contextRef,+ attempts: MAX_ATTEMPTS,+ err: lastErr instanceof Error ? lastErr.message : String(lastErr),+ }+ )+ return false}/**diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex 59a918bdf..88bf18133 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -10,6 +10,7 @@ import {resolveWorkflowControlScopes,} from '../src/hostReconciler'import type { InfrastructureTelemetryReporter } from '../src/infrastructureTelemetryReporter'+import { HostContextLogger } from '../src/logger'import { issueMcpHostRuntimeTokens } from '../src/mcpHostRuntimeTokenIssuerClient'import { HostCRD, type HostWorkflowControlScope } from '../src/types'import {@@ -2047,3 +2048,73 @@ describe('HostReconciler oauth:user-token runtime scope provisioning', () => {expect(hashInput).toContain('oauth:user-token')})})++describe('HostReconciler.frontsOAuthServer transient-blip retry (R4-M9)', () => {+ // The oauth-server probe is exercised through the REAL reconciler built by+ // createReconciler() and the REAL setHostFrontsOAuthServer setter — no+ // hand-minted value. We assert on the private method directly (rather than the+ // full reconcile) on purpose: the deployment drift guard+ // (ensureCurrentRuntimeTokenScope) re-mints the token when the scope hash+ // changes mid-reconcile, so a probe that throws once and then returns `true`+ // would be MASKED at the public issuance observable — the guard would re-mint+ // with the recovered `true` value, hiding the collapse. The boolean returned by+ // frontsOAuthServer IS the observable whose flip drives the churn this fixes.+ const callFronts = (reconciler: HostReconciler, host: HostCRD): Promise<boolean> =>+ (+ reconciler as unknown as {+ frontsOAuthServer(host: HostCRD): Promise<boolean>+ }+ ).frontsOAuthServer(host)++ it('absorbs a transient throw: probe throws once then returns true → true (blip smoothed)', async () => {+ // Regression case (T3): against the pre-fix head the throw collapses to+ // `false` and this assertion fails. With the bounded retry the second+ // attempt returns the authoritative `true`.+ const { reconciler } = createReconciler()+ let calls = 0+ reconciler.setHostFrontsOAuthServer(async () => {+ calls += 1+ if (calls === 1) throw new Error('transient apiserver read blip')+ return true+ })++ await expect(callFronts(reconciler, makeHost())).resolves.toBe(true)+ expect(calls).toBe(2)+ })++ it('does NOT retry an authoritative false: probe returns false without throwing → false, called once', async () => {+ // Control: a real scope change RETURNS false; the retry must not fire, or a+ // genuine "no longer oauth" would be smeared into a stale `true`.+ const { reconciler } = createReconciler()+ let calls = 0+ reconciler.setHostFrontsOAuthServer(async () => {+ calls += 1+ return false+ })++ await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)+ expect(calls).toBe(1)+ })++ it('fails closed after exhausting retries: probe always throws → false, N attempts, warn logged', async () => {+ const warnSpy = vi.spyOn(HostContextLogger.prototype, 'warn')+ try {+ const { reconciler } = createReconciler()+ let calls = 0+ reconciler.setHostFrontsOAuthServer(async () => {+ calls += 1+ throw new Error('sustained apiserver outage')+ })++ await expect(callFronts(reconciler, makeHost())).resolves.toBe(false)+ // Retried up to the bounded ceiling (fix uses 3 total attempts); pre-fix+ // head would call the probe exactly once.+ expect(calls).toBe(3)+ const retryWarn = warnSpy.mock.calls.find(([msg]) => String(msg).includes('after retries'))+ expect(retryWarn).toBeDefined()+ expect(retryWarn?.[1]).toMatchObject({ attempts: 3 })+ } finally {+ warnSpy.mockRestore()+ }+ })+})