Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): keep a Ready runtime deployed when the held GFS renewal fails
Alfredo Lopez98685c9d926f
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 4792a9463..77a4bf322 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.371",+ "version": "0.1.372","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.371",+ "version": "0.1.372","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 4e4821e70..ff71f8766 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.371",+ "version": "0.1.372","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 4ed7020ac..fc3087d3f 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -1874,7 +1874,11 @@ export class HostReconciler {options: BootstrapOptions = {}): Promise<RuntimeTokenProvision | null> {let lastErr: unknown = null- for (let attempt = 1; attempt <= 3; attempt++) {+ // The held GFS-only renewal runs inside a reconcile that keeps a Ready+ // runtime in place; it gets one attempt so a failing renewal never blocks+ // that reconcile on backoff. The next hold pass retries it.+ const maxAttempts = options.refreshGfsOnly ? 1 : 3+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {try {const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)if (!expectedGfsSubject) {@@ -2415,16 +2419,16 @@ export class HostReconciler {host: host.name,namespace: host.namespace,attempt,- maxAttempts: 3,+ maxAttempts,error: String(err),})- if (attempt < 3) {+ if (attempt < maxAttempts) {await new Promise(resolve => setTimeout(resolve, delayMs))}}}throw new Error(- `Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after 3 attempts: ${String(lastErr)}`+ `Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}: ${String(lastErr)}`)}@@ -5214,10 +5218,29 @@ export class HostReconciler {(lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&runtimeIsReady) {- const provision = await this.provisionRuntimeTokenRevision(host, {- targetSuspended: false,- refreshGfsOnly: true,- })+ let provision: RuntimeTokenProvision | null+ try {+ provision = await this.provisionRuntimeTokenRevision(host, {+ targetSuspended: false,+ refreshGfsOnly: true,+ })+ } catch (error) {+ // The Ready runtime keeps serving on the credential it holds; a failed+ // renewal is reported, not turned into an undeployed Host. The next+ // hold pass renews again.+ log.warn('held runtime credential renewal failed; keeping the Ready runtime', {+ host: host.name,+ namespace: host.namespace,+ err: error instanceof Error ? error.message : String(error),+ })+ this.enqueueControllerError(host, 'RuntimeCredentialRenewalFailed', error)+ this.setStatus(host.name, {+ deployed: true,+ ready: runtimeIsReady,+ message: 'Held runtime credential renewal failed; runtime kept',+ })+ return+ }if (!provision) {this.setStatus(host.name, {deployed: true,diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 12abf3866..a73f9d3cb 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -2600,6 +2600,71 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {}})+ it('keeps a Ready held runtime deployed when its GFS renewal fails, without retry backoff', async () => {+ const host = makeStatelessHost({+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },+ })+ const infrastructureTelemetryReporter = createTelemetryReporterMock()+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({+ isCommunicationChannelCacheSynced: () => false,+ infrastructureTelemetryReporter,+ })+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))+ reconciler.setHostFrontsOAuthServer(async () => false)+ // A running pod's consumed record whose GFS credential is due for renewal.+ const record = await mintedRuntimeCredentialRecord(host, {+ annotations: { 'clerum.io/gfs-token-refresh-before': '2000-01-01T00:00:00.000Z' },+ })+ const runtimeSecretName = record.metadata!.name!+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!+ coreApi.readNamespacedSecret.mockImplementation(request =>+ request.name === runtimeSecretName+ ? Promise.resolve(structuredClone(record))+ : readSecret(request)+ )+ const applied = trustedRuntimeDeployment(reconciler, host)+ applied.spec!.template!.metadata!.annotations = {+ ...applied.spec!.template!.metadata!.annotations,+ 'clerum.io/runtime-token-revision':+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],+ }+ const live = persistHostDeployment(appsApi, host, applied)+ const gfsMint = vi.mocked(mintHostGfsToken)+ const defaultGfsMint = gfsMint.getMockImplementation()!+ gfsMint.mockClear()+ gfsMint.mockImplementation(async () => {+ throw new Error('gfs token endpoint unavailable')+ })+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()+ try {+ const started = performance.now()+ await reconciler.reconcile(host)+ const elapsedMs = performance.now() - started++ // Liveness witness: the hold attempted the GFS renewal exactly once.+ expect(gfsMint).toHaveBeenCalledOnce()+ expect(elapsedMs).toBeLessThan(1000)+ expect(reconciler.getStatus(host.name)).toEqual({+ deployed: true,+ ready: true,+ message: 'Held runtime credential renewal failed; runtime kept',+ })+ const controllerErrors = vi+ .mocked(infrastructureTelemetryReporter.enqueue)+ .mock.calls.filter(([event]) => event.telemetryType === 'controller_error')+ expect(controllerErrors).toHaveLength(1)+ expect(controllerErrors[0]![0].payload).toMatchObject({+ reason_code: 'RuntimeCredentialRenewalFailed',+ status: 'failed',+ })+ expect(live().spec!.replicas).toBe(1)+ expect(live().spec!.template).toEqual(applied.spec!.template)+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()+ } finally {+ gfsMint.mockImplementation(defaultGfsMint)+ }+ })+it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {const host = makeStatelessHost({status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },