Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): keep a Ready runtime deployed when the held GFS renewal fails

Alfredo Lopez98685c9d926f
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 4792a9463..77a4bf322 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.371",
+ "version": "0.1.372",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.371",
+ "version": "0.1.372",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 4e4821e70..ff71f8766 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.371",
+ "version": "0.1.372",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 4ed7020ac..fc3087d3f 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -1874,7 +1874,11 @@ export class HostReconciler {
options: BootstrapOptions = {}
): Promise<RuntimeTokenProvision | null> {
let lastErr: unknown = null
- for (let attempt = 1; attempt <= 3; attempt++) {
+ // The held GFS-only renewal runs inside a reconcile that keeps a Ready
+ // runtime in place; it gets one attempt so a failing renewal never blocks
+ // that reconcile on backoff. The next hold pass retries it.
+ const maxAttempts = options.refreshGfsOnly ? 1 : 3
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const expectedGfsSubject = makeExpectedHostGfsSubject(host.namespace, host.name)
if (!expectedGfsSubject) {
@@ -2415,16 +2419,16 @@ export class HostReconciler {
host: host.name,
namespace: host.namespace,
attempt,
- maxAttempts: 3,
+ maxAttempts,
error: String(err),
})
- if (attempt < 3) {
+ if (attempt < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, delayMs))
}
}
}
throw new Error(
- `Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after 3 attempts: ${String(lastErr)}`
+ `Failed to ensure mcp-host-runtime-token Secret for host "${host.name}" after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}: ${String(lastErr)}`
)
}
@@ -5214,10 +5218,29 @@ export class HostReconciler {
(lifecycle.effective.state === 'active' || wakeOnRunningRuntime) &&
runtimeIsReady
) {
- const provision = await this.provisionRuntimeTokenRevision(host, {
- targetSuspended: false,
- refreshGfsOnly: true,
- })
+ let provision: RuntimeTokenProvision | null
+ try {
+ provision = await this.provisionRuntimeTokenRevision(host, {
+ targetSuspended: false,
+ refreshGfsOnly: true,
+ })
+ } catch (error) {
+ // The Ready runtime keeps serving on the credential it holds; a failed
+ // renewal is reported, not turned into an undeployed Host. The next
+ // hold pass renews again.
+ log.warn('held runtime credential renewal failed; keeping the Ready runtime', {
+ host: host.name,
+ namespace: host.namespace,
+ err: error instanceof Error ? error.message : String(error),
+ })
+ this.enqueueControllerError(host, 'RuntimeCredentialRenewalFailed', error)
+ this.setStatus(host.name, {
+ deployed: true,
+ ready: runtimeIsReady,
+ message: 'Held runtime credential renewal failed; runtime kept',
+ })
+ return
+ }
if (!provision) {
this.setStatus(host.name, {
deployed: true,
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 12abf3866..a73f9d3cb 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -2600,6 +2600,71 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
}
})
+ it('keeps a Ready held runtime deployed when its GFS renewal fails, without retry backoff', async () => {
+ const host = makeStatelessHost({
+ status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },
+ })
+ const infrastructureTelemetryReporter = createTelemetryReporterMock()
+ const { reconciler, appsApi, coreApi, customApi } = createReconciler({
+ isCommunicationChannelCacheSynced: () => false,
+ infrastructureTelemetryReporter,
+ })
+ customApi.getNamespacedCustomObject.mockImplementation(async () => hostApiObject(host))
+ reconciler.setHostFrontsOAuthServer(async () => false)
+ // A running pod's consumed record whose GFS credential is due for renewal.
+ const record = await mintedRuntimeCredentialRecord(host, {
+ annotations: { 'clerum.io/gfs-token-refresh-before': '2000-01-01T00:00:00.000Z' },
+ })
+ const runtimeSecretName = record.metadata!.name!
+ const readSecret = coreApi.readNamespacedSecret.getMockImplementation()!
+ coreApi.readNamespacedSecret.mockImplementation(request =>
+ request.name === runtimeSecretName
+ ? Promise.resolve(structuredClone(record))
+ : readSecret(request)
+ )
+ const applied = trustedRuntimeDeployment(reconciler, host)
+ applied.spec!.template!.metadata!.annotations = {
+ ...applied.spec!.template!.metadata!.annotations,
+ 'clerum.io/runtime-token-revision':
+ record.metadata!.annotations!['clerum.io/runtime-token-secret-revision'],
+ }
+ const live = persistHostDeployment(appsApi, host, applied)
+ const gfsMint = vi.mocked(mintHostGfsToken)
+ const defaultGfsMint = gfsMint.getMockImplementation()!
+ gfsMint.mockClear()
+ gfsMint.mockImplementation(async () => {
+ throw new Error('gfs token endpoint unavailable')
+ })
+ vi.mocked(issueMcpHostRuntimeTokens).mockClear()
+ try {
+ const started = performance.now()
+ await reconciler.reconcile(host)
+ const elapsedMs = performance.now() - started
+
+ // Liveness witness: the hold attempted the GFS renewal exactly once.
+ expect(gfsMint).toHaveBeenCalledOnce()
+ expect(elapsedMs).toBeLessThan(1000)
+ expect(reconciler.getStatus(host.name)).toEqual({
+ deployed: true,
+ ready: true,
+ message: 'Held runtime credential renewal failed; runtime kept',
+ })
+ const controllerErrors = vi
+ .mocked(infrastructureTelemetryReporter.enqueue)
+ .mock.calls.filter(([event]) => event.telemetryType === 'controller_error')
+ expect(controllerErrors).toHaveLength(1)
+ expect(controllerErrors[0]![0].payload).toMatchObject({
+ reason_code: 'RuntimeCredentialRenewalFailed',
+ status: 'failed',
+ })
+ expect(live().spec!.replicas).toBe(1)
+ expect(live().spec!.template).toEqual(applied.spec!.template)
+ expect(vi.mocked(issueMcpHostRuntimeTokens)).not.toHaveBeenCalled()
+ } finally {
+ gfsMint.mockImplementation(defaultGfsMint)
+ }
+ })
+
it('recovers an active held Deployment from zero after minting credentials on each pass', async () => {
const host = makeStatelessHost({
status: { lifecycle: { state: 'active', wakeHandledGeneration: 1 } },