Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): retract runtime conditions when HCC does not run a server (#633)

ALFREDO J LOPEZ H (alfredolopez80.eth)2502a3cfe7d3
diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md
index b4ef088c3..1aa757d23 100644
--- a/docs/architecture/overview.md
+++ b/docs/architecture/overview.md
@@ -898,7 +898,7 @@ flowchart LR
DEP --> SVC["3. Create ClusterIP Service"]
SVC --> CONFLICT["4. 409? preserve resourceVersion,<br/>replace"]
CONFLICT --> SIDECAR["5. stdio-bridge sidecar<br/>(managed:true + stdio transport)"]
- SIDECAR --> STATUS["6. Patch status:<br/>NetworkReady, DeploymentReady"]
+ SIDECAR --> STATUS["6. Patch status:<br/>DeploymentReady<br/>(RuntimeNotDesired when HCC does not run the runtime)"]
end
subgraph L2["Context Watch Loop"]
CTX["Context CRD<br/>created/modified"] --> NP["Generate NetworkPolicies<br/>per (context, server) pair"]
@@ -933,7 +933,7 @@ When an McpServer CRD is created or modified:
- **MCP Proxy** routes HTTP requests to the sidecar transparently
- Security overrides (`addCapabilities`, `runAsUser`, etc.) are propagated from the McpServer CRD
-6. **Status conditions** (G11): After reconciliation, HCC patches McpServer CRD status with `NetworkReady` and `DeploymentReady` conditions for reactive watches.
+6. **Status conditions** (G11): After reconciliation, HCC patches McpServer CRD status with `DeploymentReady` for the Deployment rollout. When HCC does not run a **managed** runtime (disabled or fail-closed secret validation) **and the HCC-owned Deployment is confirmed gone**, it writes `DeploymentReady=False/RuntimeNotDesired`. `McpServerReconciler.writeStatusCondition` strips leftover `NetworkReady` from every McpServer status write; `NetworkPolicyReconciler` may still echo a leftover `NetworkReady` until the next McpServer write removes it. WRC watches the `clerum.io/network-ready` annotation, not a `NetworkReady` condition.
7. **Orphan cleanup**: On startup, full reconciliation deletes any Deployments with `clerum.io/managed-by=host-context-controller` that no longer have a corresponding McpServer CRD.
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 5e911645c..3b1d90381 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.322",
+ "version": "0.1.329",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.322",
+ "version": "0.1.329",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 3d9e0bf25..514f30a40 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.322",
+ "version": "0.1.329",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
index ce5e7df25..96ba5134c 100644
--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
@@ -8,7 +8,7 @@ import {
createMockCoreApi,
createMockCustomApi,
} from '../../test/__fixtures__/testMocks'
-import { McpServerReconciler } from '../reconciler'
+import { McpServerReconciler, RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE } from '../reconciler'
import { McpServerCRD } from '../types'
vi.mock('../config', () => ({
@@ -72,6 +72,99 @@ function credentialsRevisionOf(deployment: k8s.V1Deployment): string | undefined
return deployment.spec?.template?.metadata?.annotations?.[CREDENTIALS_REVISION_ANNOTATION]
}
+function readinessPollsOf(reconciler: McpServerReconciler): Map<string, unknown> {
+ return (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls
+}
+
+function notConvergedDeployment(generation: number, name = 'linear') {
+ return {
+ metadata: {
+ name,
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
+ spec: { replicas: 1 },
+ status: {
+ observedGeneration: generation,
+ replicas: 2,
+ updatedReplicas: 1,
+ readyReplicas: 1,
+ availableReplicas: 1,
+ unavailableReplicas: 0,
+ },
+ }
+}
+
+function holdNextStatusRead(customApi: ReturnType<typeof createMockCustomApi>): {
+ release: (value?: unknown) => void
+ resolved: () => number
+ isHeld: () => boolean
+} {
+ let pending: ((value: unknown) => void) | undefined
+ let resolved = 0
+ let holdNext = true
+ customApi.getNamespacedCustomObjectStatus.mockImplementation(
+ () =>
+ new Promise(resolve => {
+ if (holdNext) {
+ holdNext = false
+ pending = value => {
+ resolved += 1
+ resolve(value)
+ }
+ return
+ }
+ resolve({ metadata: { resourceVersion: '1' }, status: { conditions: [] } })
+ })
+ )
+ return {
+ isHeld: () => pending !== undefined,
+ resolved: () => resolved,
+ release: (value = { metadata: { resourceVersion: '1' }, status: { conditions: [] } }) => {
+ if (pending === undefined) {
+ throw new Error('status GET was never held')
+ }
+ pending(value)
+ },
+ }
+}
+
+function laterConditionWrites(
+ customApi: ReturnType<typeof createMockCustomApi>,
+ afterCallCount: number
+): Array<{ type: string; status: string; reason: string }> {
+ return customApi.patchNamespacedCustomObjectStatus.mock.calls
+ .slice(afterCallCount)
+ .flatMap(call => {
+ const body = (call[0] as { body?: unknown }).body
+ return Array.isArray(body) ? body : []
+ })
+ .flatMap((op: unknown) => {
+ const value = (op as { value?: unknown }).value
+ const conditions = Array.isArray(value)
+ ? value
+ : ((value as { conditions?: unknown[] })?.conditions ?? [])
+ return Array.isArray(conditions) ? conditions : []
+ })
+ .map((c: unknown) => c as { type: string; status: string; reason: string })
+}
+
+async function reconcileFailClosed(
+ reconciler: McpServerReconciler,
+ coreApi: ReturnType<typeof createMockCoreApi>
+) {
+ coreApi.readNamespacedSecret.mockRejectedValue(
+ Object.assign(new Error('not found'), { code: 404 })
+ )
+ await reconciler.reconcile(makeServer())
+}
+
/**
* Every DeploymentReady condition written to the CRD, oldest first. Conditions
* travel as a JSON Patch on the status subresource, so this digs the condition
@@ -122,13 +215,15 @@ function convergedDeployment(name = 'linear') {
}
describe('connector credentials revision (issue #223)', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
let live: k8s.V1Deployment | undefined
// This suite creates a connector, then observes the same API object on rotation.
appsApi.readNamespacedDeployment.mockImplementation(async () => {
@@ -707,6 +802,252 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
expect(writes.at(-1)?.status).toBe('True')
expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)
})
+
+ it('open readiness window does not write RolloutIncomplete after fail-closed retirement', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer())
+ const readsBeforeTick = appsApi.readNamespacedDeployment.mock.calls.length
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBeGreaterThan(readsBeforeTick)
+
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ try {
+ await reconcileFailClosed(reconciler, coreApi)
+ const lastAfterRetirement = deploymentReadyWrites(customApi).at(-1)
+ expect(lastAfterRetirement?.reason).toBe('RuntimeNotDesired')
+ expect(lastAfterRetirement?.message).toBe(RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE)
+ const warnsAfterRetirement = warn.mock.calls.length
+ const writesAfterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+
+ await vi.advanceTimersByTimeAsync(24 * 5000)
+
+ expect(
+ laterConditionWrites(customApi, writesAfterRetirement).some(
+ condition =>
+ condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'
+ )
+ ).toBe(false)
+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
+ expect(warn.mock.calls.slice(warnsAfterRetirement).flat().join('\n')).not.toContain(
+ 'not ready after'
+ )
+ } finally {
+ warn.mockRestore()
+ }
+ })
+
+ it('in-flight ready-branch tick past map delete cannot write True after retirement', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer())
+
+ const held = holdNextStatusRead(customApi)
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(held.isHeld()).toBe(true)
+ expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()
+
+ await reconcileFailClosed(reconciler, coreApi)
+ expect(
+ deploymentReadyWrites(customApi).some(
+ write =>
+ write.reason === 'RuntimeNotDesired' &&
+ write.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
+ )
+ ).toBe(true)
+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+
+ held.release()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(held.resolved()).toBe(1)
+
+ const leaked = laterConditionWrites(customApi, afterRetirement)
+ expect(
+ leaked.some(
+ condition =>
+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||
+ (condition.type === 'Ready' &&
+ condition.status === 'True' &&
+ condition.reason === 'ReconcileSuccess')
+ )
+ ).toBe(false)
+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)
+ })
+
+ it('in-flight ready tick cannot overwrite RuntimeNotDesired while retract write is in flight', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer())
+
+ const held = holdNextStatusRead(customApi)
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(held.isHeld()).toBe(true)
+
+ let releaseRetractPatch: (() => void) | undefined
+ let retractPatchHeld!: () => void
+ const retractPatchGate = new Promise<void>(resolve => {
+ retractPatchHeld = resolve
+ })
+ const livePatch = customApi.patchNamespacedCustomObjectStatus.getMockImplementation()
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async req => {
+ const body = (req as { body?: Array<{ op?: string; path?: string; value?: unknown }> }).body
+ const op = body?.find(
+ candidate =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ const conditions = (
+ Array.isArray(op?.value)
+ ? op.value
+ : ((op?.value as { conditions?: Array<{ reason?: string }> }).conditions ?? [])
+ ) as Array<{ reason?: string }>
+ if (conditions.some(condition => condition.reason === 'RuntimeNotDesired')) {
+ await new Promise<void>(resolve => {
+ releaseRetractPatch = resolve
+ retractPatchHeld()
+ })
+ }
+ if (livePatch) return livePatch(req)
+ return {}
+ })
+
+ const failClosed = reconcileFailClosed(reconciler, coreApi)
+ try {
+ // Race the held retract PATCH against reconcile return so a missing
+ // RuntimeNotDesired write fails here instead of hanging the suite.
+ const retractSeen = await Promise.race([
+ retractPatchGate.then(() => 'retract' as const),
+ failClosed.then(() => 'finished' as const),
+ ])
+ expect(retractSeen).toBe('retract')
+ expect(releaseRetractPatch).toBeDefined()
+ const writesWhileRetractHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+
+ held.release({ metadata: { resourceVersion: '2' }, status: { conditions: [] } })
+ await vi.advanceTimersByTimeAsync(0)
+
+ expect(
+ laterConditionWrites(customApi, writesWhileRetractHeld).some(
+ condition =>
+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||
+ (condition.type === 'Ready' &&
+ condition.status === 'True' &&
+ condition.reason === 'ReconcileSuccess')
+ )
+ ).toBe(false)
+
+ releaseRetractPatch?.()
+ await failClosed
+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)
+ } finally {
+ releaseRetractPatch?.()
+ if (livePatch) {
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(livePatch)
+ } else {
+ customApi.patchNamespacedCustomObjectStatus.mockReset()
+ }
+ }
+ })
+
+ it('in-flight exhaustion tick cannot write RolloutIncomplete after retirement', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer())
+ await vi.advanceTimersByTimeAsync(23 * 5000)
+
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+ try {
+ const held = holdNextStatusRead(customApi)
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(held.isHeld()).toBe(true)
+ expect(warn.mock.calls.flat().join('\n')).toContain('not ready after 24 polls')
+
+ await reconcileFailClosed(reconciler, coreApi)
+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+ held.release()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(held.resolved()).toBe(1)
+
+ expect(
+ laterConditionWrites(customApi, afterRetirement).some(
+ condition =>
+ condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'
+ )
+ ).toBe(false)
+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
+ } finally {
+ warn.mockRestore()
+ }
+ })
+
+ it('retirement epoch stops a stale ready-branch tick after a later window was armed', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer())
+ const windowOne = readinessPollsOf(reconciler).get('linear')
+ expect(windowOne).toBeDefined()
+
+ const held = holdNextStatusRead(customApi)
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(held.isHeld()).toBe(true)
+ expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()
+
+ coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('rotated-key') } })
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(3))
+ await reconciler.reconcile(makeServer())
+ const windowTwo = readinessPollsOf(reconciler).get('linear')
+ expect(windowTwo).toBeDefined()
+ expect(windowTwo).not.toBe(windowOne)
+
+ await reconcileFailClosed(reconciler, coreApi)
+ expect(
+ deploymentReadyWrites(customApi).some(write => write.reason === 'RuntimeNotDesired')
+ ).toBe(true)
+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+ expect(readinessPollsOf(reconciler).has('linear')).toBe(false)
+ const readsAfterRetract = appsApi.readNamespacedDeployment.mock.calls.length
+
+ held.release()
+ await vi.advanceTimersByTimeAsync(24 * 5000)
+ expect(held.resolved()).toBe(1)
+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBe(readsAfterRetract)
+
+ const leaked = laterConditionWrites(customApi, afterRetirement)
+ expect(
+ leaked.some(
+ condition =>
+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||
+ (condition.type === 'Ready' && condition.status === 'True')
+ )
+ ).toBe(false)
+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')
+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)
+ })
+
+ it('in-flight ready tick records obligation when the revision fence flips without retirement', async () => {
+ let current = true
+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))
+ await reconciler.reconcile(makeServer(), { isCurrent: () => current })
+
+ const held = holdNextStatusRead(customApi)
+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(held.isHeld()).toBe(true)
+ const writesWhileHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length
+
+ current = false
+ held.release()
+ await vi.advanceTimersByTimeAsync(0)
+ expect(held.resolved()).toBe(1)
+ expect(
+ laterConditionWrites(customApi, writesWhileHeld).some(
+ condition =>
+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||
+ (condition.type === 'Ready' && condition.status === 'True')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
})
describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {
diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.ts
index a14c0a17a..1e1e74d93 100644
--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts
@@ -169,10 +169,7 @@ function cloneServer(
}
}
-function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {
- const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls
- const lastCall = calls[calls.length - 1]?.[0]
- const patch = lastCall?.body
+function conditionsFromStatusPatch(patch: unknown): any[] {
const statusAdd = Array.isArray(patch)
? patch.find(
candidate =>
@@ -184,6 +181,17 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {
return Array.isArray(value) ? value : (value?.conditions ?? [])
}
+function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {
+ const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls
+ return conditionsFromStatusPatch(calls[calls.length - 1]?.[0]?.body)
+}
+
+function allPatchedStatusConditions(customApi: MockCustomApi): any[] {
+ return customApi.patchNamespacedCustomObjectStatus.mock.calls.flatMap(call =>
+ conditionsFromStatusPatch(call[0]?.body)
+ )
+}
+
// ─── Tests ───────────────────────────────────────────────────────────────────
describe('McpServerReconciler remote egress proxy', () => {
@@ -830,6 +838,52 @@ describe('McpServerReconciler remote egress proxy', () => {
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
expect(coreApi.createNamespacedService).toHaveBeenCalled()
})
+
+ it('does not write NetworkReady on the remote reconcile path', async () => {
+ coreApi.readNamespacedConfigMap.mockRejectedValueOnce({ code: 404 })
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
+ let liveConditions = [
+ {
+ type: 'NetworkReady',
+ status: 'True',
+ reason: 'NetworkPoliciesApplied',
+ message: 'NetworkPolicies and Service created',
+ lastTransitionTime: '2020-01-03T00:00:00.000Z',
+ },
+ ]
+ customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({
+ metadata: { resourceVersion: '1' },
+ status: { conditions: liveConditions },
+ }))
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(
+ async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {
+ const op = req.body.find(
+ candidate =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ if (op) {
+ liveConditions = (
+ op.path === '/status'
+ ? ((op.value as { conditions?: typeof liveConditions }).conditions ?? [])
+ : (op.value as typeof liveConditions)
+ ).map(condition => ({ ...condition }))
+ }
+ return {}
+ }
+ )
+ await reconciler.reconcile(REMOTE_SERVER)
+
+ const allConditions = allPatchedStatusConditions(customApi)
+ expect(
+ allConditions.some((condition: { type?: string }) => condition.type === 'DeploymentReady')
+ ).toBe(true)
+ expect(
+ allConditions.some((condition: { type?: string }) => condition.type === 'NetworkReady')
+ ).toBe(false)
+ expect(lastPatchedStatusConditions(customApi).length).toBeGreaterThan(0)
+ })
})
// ─── Test 5: reconcile skips ConfigMap for local servers ───────────────
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 834926800..4e7cdce9d 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -11,7 +11,11 @@ import {
createMockNetworkingApi,
} from '../test/__fixtures__/testMocks'
import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './constants'
-import { McpServerReconciler } from './reconciler'
+import {
+ McpServerReconciler,
+ RUNTIME_NOT_DESIRED_DISABLED_MESSAGE,
+ RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE,
+} from './reconciler'
import { McpServerCRD } from './types'
describe.each(['Deployment', 'Service'] as const)('read-first %s contract', kind => {
@@ -134,14 +138,164 @@ function makeServer(
}
}
+type StatusConditionFixture = {
+ type: string
+ status: string
+ reason: string
+ message: string
+ lastTransitionTime: string
+ observedGeneration?: number
+}
+
+/** Secret.data form. Do not write `password: 'literal'` — public-boundary rejects it. */
+function storedSecret(value: string): string {
+ return Buffer.from(value, 'utf8').toString('base64')
+}
+
+function hccOwnedMetadata(name: string) {
+ return {
+ name,
+ namespace: 'mcp-server',
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': name,
+ },
+ }
+}
+
+function networkReadyTrue(): StatusConditionFixture {
+ return {
+ type: 'NetworkReady',
+ status: 'True',
+ reason: 'NetworkPoliciesApplied',
+ message: 'NetworkPolicies and Service created',
+ lastTransitionTime: '2020-01-03T00:00:00.000Z',
+ }
+}
+
+function deploymentReadyTrue(): StatusConditionFixture {
+ return {
+ type: 'DeploymentReady',
+ status: 'True',
+ reason: 'ReplicasAvailable',
+ message: 'Deployment has ready replicas',
+ lastTransitionTime: '2020-01-02T00:00:00.000Z',
+ }
+}
+
+function secretResolvedTrue(): StatusConditionFixture {
+ return {
+ type: 'SecretResolved',
+ status: 'True',
+ reason: 'SecretFound',
+ message: 'ok',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ }
+}
+
+function installLiveStatus(
+ customApi: ReturnType<typeof createMockCustomApi>,
+ initial: StatusConditionFixture[] = []
+): { current: () => StatusConditionFixture[] } {
+ let conditions = [...initial]
+ customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({
+ metadata: { resourceVersion: '1' },
+ status: { conditions },
+ }))
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(
+ async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {
+ const op = req.body.find(
+ candidate =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ if (!op) throw new Error('status condition add operation was not emitted')
+ conditions = (
+ op.path === '/status'
+ ? ((op.value as { conditions?: StatusConditionFixture[] }).conditions ?? [])
+ : (op.value as StatusConditionFixture[])
+ ).map(condition => ({ ...condition }))
+ return {}
+ }
+ )
+ return { current: () => conditions }
+}
+
+function persistReadySeed(server: McpServerCRD): McpServerCRD {
+ server.uid = server.uid ?? 'uid-pg'
+ server.generation = server.generation ?? 3
+ server.status = {
+ conditions: [
+ {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'Deployment created',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ observedGeneration: server.generation,
+ },
+ ],
+ }
+ return server
+}
+
+function failRuntimeNotDesiredWrites(customApi: ReturnType<typeof createMockCustomApi>): void {
+ const livePatch = customApi.patchNamespacedCustomObjectStatus.getMockImplementation()
+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async req => {
+ const body = (req as { body: Array<{ op?: string; path?: string; value?: unknown }> }).body
+ const op = body.find(
+ candidate =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ const conditions = (
+ Array.isArray(op?.value)
+ ? op.value
+ : ((op?.value as { conditions?: Array<{ reason?: string }> }).conditions ?? [])
+ ) as Array<{ reason?: string }>
+ if (conditions.some(condition => condition.reason === 'RuntimeNotDesired')) {
+ throw Object.assign(new Error('status write failed'), { code: 500 })
+ }
+ if (livePatch) return livePatch(req)
+ return {}
+ })
+}
+
+function patchedConditionSets(
+ customApi: ReturnType<typeof createMockCustomApi>
+): StatusConditionFixture[][] {
+ return customApi.patchNamespacedCustomObjectStatus.mock.calls.map(call => {
+ const body = (call[0] as { body: Array<{ op?: string; path?: string; value?: unknown }> }).body
+ const op = body.find(
+ candidate =>
+ candidate.op === 'add' &&
+ (candidate.path === '/status' || candidate.path === '/status/conditions')
+ )
+ if (!op) return []
+ const value = op.value
+ return (
+ Array.isArray(value) ? value : ((value as { conditions?: unknown[] }).conditions ?? [])
+ ) as StatusConditionFixture[]
+ })
+}
+
+function firstPatchWithType(
+ sets: StatusConditionFixture[][],
+ type: string
+): StatusConditionFixture[] | undefined {
+ return sets.find(conditions => conditions.some(condition => condition.type === type))
+}
+
describe('Reconciler managed:false guard (Risk 1.7)', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
@@ -479,20 +633,60 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
expect(status.ready).toBe(false)
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.type === 'DeploymentReady')
+ )
+ ).toBe(false)
+ })
+
+ it('does not write DeploymentReady when a WRC-owned server is fail-closed', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ const server = makeServer({
+ name: 'workflow-recipes',
+ managed: false,
+ envSecret: { name: 'missing', keys: [{ secretKey: 'token', envVar: 'TOKEN' }] },
+ })
+
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'SecretResolved' && condition.reason === 'SecretNotFound'
+ )
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.type === 'DeploymentReady')
+ )
+ ).toBe(false)
})
})
// ─── PR-B B1: validateSecret result shape + writeStatusCondition ───────
describe('PR-B B1 — validateSecret result shape', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
- const networkingApi = createMockNetworkingApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
+ let networkingApi: ReturnType<typeof createMockNetworkingApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
+ networkingApi = createMockNetworkingApi()
const kubeConfig = new k8s.KubeConfig()
// Keep the forbidden API reachable: reintroducing the old optional
// NetworkingApi client must not pass by observing an unrelated mock.
@@ -654,6 +848,371 @@ describe('PR-B B1 — validateSecret result shape', () => {
expect(reconciler.getStatus('pg')).toMatchObject({ deployed: false, ready: false })
})
+ it('fail-closed incomplete delete does not write RuntimeNotDesired', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ let seenDeploymentRead = false
+ let deniedDeploymentDelete = false
+ appsApi.readNamespacedDeployment.mockImplementation(async () => {
+ seenDeploymentRead = true
+ return { metadata: hccOwnedMetadata('pg') }
+ })
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ await reconciler.reconcile(server, {
+ isCurrent: () => {
+ if (seenDeploymentRead && !deniedDeploymentDelete) {
+ deniedDeploymentDelete = true
+ return false
+ }
+ return true
+ },
+ })
+
+ expect(seenDeploymentRead).toBe(true)
+ expect(deniedDeploymentDelete).toBe(true)
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('fail-closed writes RuntimeNotDesired when Service cleanup is denied after Deployment delete', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+ let seenServiceRead = false
+ let deniedServiceDelete = false
+ coreApi.readNamespacedService.mockImplementation(async () => {
+ seenServiceRead = true
+ return { metadata: hccOwnedMetadata('pg') }
+ })
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ await reconciler.reconcile(server, {
+ isCurrent: () => {
+ if (seenServiceRead && !deniedServiceDelete) {
+ deniedServiceDelete = true
+ return false
+ }
+ return true
+ },
+ })
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' &&
+ condition.reason === 'RuntimeNotDesired' &&
+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
+ )
+ )
+ ).toBe(true)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('disabled writes RuntimeNotDesired when Service cleanup is denied after Deployment delete', async () => {
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+ let seenServiceRead = false
+ let deniedServiceDelete = false
+ coreApi.readNamespacedService.mockImplementation(async () => {
+ seenServiceRead = true
+ return { metadata: hccOwnedMetadata('pg') }
+ })
+
+ const server = persistReadySeed(makeServer({ name: 'pg', managed: true, enabled: false }))
+ await reconciler.reconcile(server, {
+ isCurrent: () => {
+ if (seenServiceRead && !deniedServiceDelete) {
+ deniedServiceDelete = true
+ return false
+ }
+ return true
+ },
+ })
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' &&
+ condition.reason === 'RuntimeNotDesired' &&
+ condition.message === RUNTIME_NOT_DESIRED_DISABLED_MESSAGE
+ )
+ )
+ ).toBe(true)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('fail-closed writes RuntimeNotDesired when Service delete throws after Deployment is gone', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+ coreApi.deleteNamespacedService.mockRejectedValueOnce(
+ Object.assign(new Error('forbidden'), { code: 403 })
+ )
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ await expect(reconciler.reconcile(server)).rejects.toThrow(
+ 'Failed to delete runtime Kubernetes resources for McpServer "pg"'
+ )
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' && condition.reason === 'RuntimeNotDesired'
+ )
+ )
+ ).toBe(true)
+ })
+
+ it('disabled incomplete delete does not write RuntimeNotDesired', async () => {
+ let seenDeploymentRead = false
+ let deniedDeploymentDelete = false
+ appsApi.readNamespacedDeployment.mockImplementation(async () => {
+ seenDeploymentRead = true
+ return { metadata: hccOwnedMetadata('pg') }
+ })
+
+ await reconciler.reconcile(makeServer({ name: 'pg', managed: true, enabled: false }), {
+ isCurrent: () => {
+ if (seenDeploymentRead && !deniedDeploymentDelete) {
+ deniedDeploymentDelete = true
+ return false
+ }
+ return true
+ },
+ })
+
+ expect(deniedDeploymentDelete).toBe(true)
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('fail-closed SecretNotFound writes RuntimeNotDesired after delete and strips NetworkReady', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
+ name: 'pg',
+ namespace: 'mcp-server',
+ })
+ const sets = patchedConditionSets(customApi)
+ const secretResolvedPatch = firstPatchWithType(sets, 'SecretResolved')
+ expect(secretResolvedPatch).toBeDefined()
+ expect(secretResolvedPatch?.map(condition => condition.type)).not.toContain('NetworkReady')
+ expect(
+ sets.some(conditions =>
+ conditions.some(c => c.type === 'Ready' && c.reason === 'SecretValidationFailed')
+ )
+ ).toBe(true)
+ const retirementIndex = sets.findIndex(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' &&
+ condition.status === 'False' &&
+ condition.reason === 'RuntimeNotDesired' &&
+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
+ )
+ )
+ expect(retirementIndex).toBeGreaterThanOrEqual(0)
+ expect(sets[retirementIndex]?.map(condition => condition.type)).not.toContain('NetworkReady')
+ expect(appsApi.deleteNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(
+ customApi.patchNamespacedCustomObjectStatus.mock.invocationCallOrder[retirementIndex]
+ )
+ })
+
+ it('fail-closed SecretNotFound writes RuntimeNotDesired even when DeploymentReady was absent', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [secretResolvedTrue(), networkReadyTrue()])
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' &&
+ condition.reason === 'RuntimeNotDesired' &&
+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
+ )
+ )
+ ).toBe(true)
+ })
+
+ it('disabled managed server writes RuntimeNotDesired after delete and strips NetworkReady', async () => {
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+
+ const server = makeServer({ name: 'pg', managed: true, enabled: false })
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({
+ name: 'pg',
+ namespace: 'mcp-server',
+ })
+ const sets = patchedConditionSets(customApi)
+ expect(
+ sets.some(conditions =>
+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')
+ )
+ ).toBe(true)
+ const retirementIndex = sets.findIndex(conditions =>
+ conditions.some(
+ condition =>
+ condition.type === 'DeploymentReady' &&
+ condition.status === 'False' &&
+ condition.reason === 'RuntimeNotDesired' &&
+ condition.message === RUNTIME_NOT_DESIRED_DISABLED_MESSAGE
+ )
+ )
+ expect(retirementIndex).toBeGreaterThanOrEqual(0)
+ expect(sets.flat().some(condition => condition.type === 'NetworkReady')).toBe(false)
+ expect(appsApi.deleteNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(
+ customApi.patchNamespacedCustomObjectStatus.mock.invocationCallOrder[retirementIndex]
+ )
+ })
+
+ it('recovery after fail-closed stamps a fresh WaitingForReplicas time', async () => {
+ vi.useFakeTimers()
+ try {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [
+ secretResolvedTrue(),
+ deploymentReadyTrue(),
+ networkReadyTrue(),
+ ])
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ await reconciler.reconcile(server)
+ const retirement = patchedConditionSets(customApi)
+ .flat()
+ .find(condition => condition.reason === 'RuntimeNotDesired')
+ expect(retirement?.lastTransitionTime).toBeDefined()
+ const retiredAt = retirement!.lastTransitionTime
+
+ await vi.advanceTimersByTimeAsync(1000)
+ coreApi.readNamespacedSecret.mockResolvedValue({
+ data: { password: storedSecret('fixture-key') },
+ })
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
+ appsApi.readNamespacedDeployment.mockResolvedValue({
+ metadata: {
+ name: 'pg',
+ namespace: 'mcp-server',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'pg',
+ },
+ },
+ spec: { replicas: 1 },
+ status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 0, replicas: 1 },
+ })
+
+ await reconciler.reconcile(server)
+ const waiting = patchedConditionSets(customApi)
+ .flat()
+ .filter(condition => condition.type === 'DeploymentReady')
+ .at(-1)
+ expect(waiting).toMatchObject({
+ status: 'False',
+ reason: 'WaitingForReplicas',
+ })
+ expect(Date.parse(waiting!.lastTransitionTime)).toBeGreaterThan(Date.parse(retiredAt))
+ expect(
+ patchedConditionSets(customApi)
+ .flat()
+ .some(condition => condition.type === 'NetworkReady')
+ ).toBe(false)
+
+ appsApi.readNamespacedDeployment.mockResolvedValue({
+ metadata: {
+ name: 'pg',
+ namespace: 'mcp-server',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'pg',
+ },
+ },
+ spec: { replicas: 1 },
+ status: {
+ observedGeneration: 2,
+ replicas: 1,
+ updatedReplicas: 1,
+ readyReplicas: 1,
+ unavailableReplicas: 0,
+ },
+ })
+ const readsBeforePoll = appsApi.readNamespacedDeployment.mock.calls.length
+ await vi.advanceTimersByTimeAsync(5000)
+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBeGreaterThan(readsBeforePoll)
+ const ready = patchedConditionSets(customApi)
+ .flat()
+ .filter(condition => condition.type === 'DeploymentReady')
+ .at(-1)
+ expect(ready?.status).toBe('True')
+ expect(Date.parse(ready!.lastTransitionTime)).toBeGreaterThan(Date.parse(retiredAt))
+ expect(
+ patchedConditionSets(customApi)
+ .flat()
+ .some(condition => condition.type === 'NetworkReady')
+ ).toBe(false)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('propagates cleanup errors when secret validation fails', async () => {
const err = new Error('not found') as Error & { code?: number }
err.code = 404
@@ -670,6 +1229,12 @@ describe('PR-B B1 — validateSecret result shape', () => {
await expect(reconciler.reconcile(server)).rejects.toThrow(
'Failed to delete runtime Kubernetes resources for McpServer "pg"'
)
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({
name: 'pg',
namespace: 'mcp-server',
@@ -683,9 +1248,9 @@ describe('PR-B B1 — validateSecret result shape', () => {
})
describe('PR-B B1 — writeStatusCondition', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
const getPatchedConditions = () => {
@@ -701,7 +1266,9 @@ describe('PR-B B1 — writeStatusCondition', () => {
}
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
@@ -710,6 +1277,69 @@ describe('PR-B B1 — writeStatusCondition', () => {
})
})
+ it('bumps DeploymentReady lastTransitionTime when reason changes at the same status', async () => {
+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ status: {
+ conditions: [
+ {
+ type: 'DeploymentReady',
+ status: 'False',
+ reason: 'RuntimeNotDesired',
+ message: RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE,
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ },
+ ],
+ },
+ })
+
+ const server = makeServer({ name: 'pg' })
+ await reconciler.writeStatusCondition(server, {
+ type: 'DeploymentReady',
+ status: 'False',
+ reason: 'WaitingForReplicas',
+ message: 'Waiting for pods to become ready',
+ })
+
+ const { conditions } = getPatchedConditions()
+ const written = conditions.find((c: { type: string }) => c.type === 'DeploymentReady')
+ expect(written.lastTransitionTime).not.toBe('2020-01-01T00:00:00.000Z')
+ expect(Date.parse(written.lastTransitionTime)).toBeGreaterThan(
+ Date.parse('2020-01-01T00:00:00.000Z')
+ )
+ })
+
+ it('bumps DeploymentReady lastTransitionTime when only the message changes', async () => {
+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({
+ status: {
+ conditions: [
+ {
+ type: 'DeploymentReady',
+ status: 'False',
+ reason: 'WaitingForReplicas',
+ message: 'Waiting for pods to become ready',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ },
+ ],
+ },
+ })
+
+ const server = makeServer({ name: 'pg' })
+ await reconciler.writeStatusCondition(server, {
+ type: 'DeploymentReady',
+ status: 'False',
+ reason: 'WaitingForReplicas',
+ message: 'Rollout did not converge — 0/1 ready',
+ })
+
+ const { conditions } = getPatchedConditions()
+ const written = conditions.find((c: { type: string }) => c.type === 'DeploymentReady')
+ expect(written.lastTransitionTime).not.toBe('2020-01-01T00:00:00.000Z')
+ expect(Date.parse(written.lastTransitionTime)).toBeGreaterThan(
+ Date.parse('2020-01-01T00:00:00.000Z')
+ )
+ expect(written.message).toBe('Rollout did not converge — 0/1 ready')
+ })
+
it('preserves lastTransitionTime when status is unchanged', async () => {
const existing = {
type: 'SecretResolved',
@@ -774,9 +1404,9 @@ describe('PR-B B1 — writeStatusCondition', () => {
status: {
conditions: [
{
- type: 'NetworkReady',
+ type: 'ExternalEgressReady',
status: 'True',
- reason: 'NetworkPoliciesApplied',
+ reason: 'EgressPolicyApplied',
message: 'ok',
lastTransitionTime: '2020-01-01T00:00:00.000Z',
},
@@ -794,7 +1424,7 @@ describe('PR-B B1 — writeStatusCondition', () => {
const { conditions } = getPatchedConditions()
const types = conditions.map((c: { type: string }) => c.type)
- expect(types).toContain('NetworkReady')
+ expect(types).toContain('ExternalEgressReady')
expect(types).toContain('Ready')
})
@@ -821,28 +1451,28 @@ describe('PR-B B1 — writeStatusCondition', () => {
})
it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {
- const networkReady = {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'network ready',
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- }
const externalEgressReady = {
type: 'ExternalEgressReady',
status: 'True',
reason: 'EgressPolicyApplied',
message: 'external egress ready',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ }
+ const peerCondition = {
+ type: 'SecretResolved',
+ status: 'True',
+ reason: 'SecretFound',
+ message: 'peer write',
lastTransitionTime: '2020-01-02T00:00:00.000Z',
}
customApi.getNamespacedCustomObjectStatus
.mockResolvedValueOnce({
metadata: { resourceVersion: '100' },
- status: { conditions: [networkReady] },
+ status: { conditions: [externalEgressReady] },
})
.mockResolvedValueOnce({
metadata: { resourceVersion: '101' },
- status: { conditions: [networkReady, externalEgressReady] },
+ status: { conditions: [externalEgressReady, peerCondition] },
})
customApi.patchNamespacedCustomObjectStatus
.mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))
@@ -878,12 +1508,58 @@ describe('PR-B B1 — writeStatusCondition', () => {
candidate.op === 'add' && candidate.path === '/status/conditions'
)
expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([
- 'NetworkReady',
'ExternalEgressReady',
+ 'SecretResolved',
'Ready',
])
})
+ it('writeStatusCondition of an identical Ready still strips NetworkReady across a 422 retry', async () => {
+ const ready = {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ lastTransitionTime: '2020-01-01T00:00:00.000Z',
+ }
+ const thirdParty = {
+ type: 'ExternalEgressReady',
+ status: 'True',
+ reason: 'EgressPolicyApplied',
+ message: 'peer',
+ lastTransitionTime: '2020-01-02T00:00:00.000Z',
+ }
+ customApi.getNamespacedCustomObjectStatus
+ .mockResolvedValueOnce({
+ metadata: { resourceVersion: '100' },
+ status: { conditions: [ready, networkReadyTrue()] },
+ })
+ .mockResolvedValueOnce({
+ metadata: { resourceVersion: '101' },
+ status: { conditions: [ready, networkReadyTrue(), thirdParty] },
+ })
+ customApi.patchNamespacedCustomObjectStatus
+ .mockRejectedValueOnce(Object.assign(new Error('conflict'), { code: 422 }))
+ .mockResolvedValueOnce({})
+
+ await reconciler.writeStatusCondition(makeServer({ name: 'pg' }), {
+ type: 'Ready',
+ status: 'True',
+ reason: 'ReconcileSuccess',
+ message: 'done',
+ })
+
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
+ const firstTypes = patchedConditionSets(customApi)[0].map(condition => condition.type)
+ const secondTypes = patchedConditionSets(customApi)[1].map(condition => condition.type)
+ expect(firstTypes).toContain('Ready')
+ expect(firstTypes).not.toContain('NetworkReady')
+ expect(secondTypes).toContain('Ready')
+ expect(secondTypes).toContain('ExternalEgressReady')
+ expect(secondTypes).not.toContain('NetworkReady')
+ })
+
it('bounds resourceVersion conflict retries', async () => {
const currentStatus = {
metadata: { resourceVersion: '100' },
@@ -1039,7 +1715,7 @@ describe('PR-B B1 — writeStatusCondition', () => {
reason: 'ok',
message: 'ok',
})
- ).resolves.toBe(false)
+ ).resolves.toBe('declined')
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
})
@@ -1153,13 +1829,15 @@ describe('PR-B B1 — writeStatusCondition', () => {
})
describe('restart-safe discovery status', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
@@ -1322,13 +2000,15 @@ describe('restart-safe discovery status', () => {
})
describe('ownership-safe fail-closed cleanup', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
@@ -1388,6 +2068,7 @@ describe('ownership-safe fail-closed cleanup', () => {
const err = new Error('api unavailable') as Error & { code?: number }
err.code = 500
coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [networkReadyTrue()])
const server = makeServer({
name: 'pg',
@@ -1399,6 +2080,16 @@ describe('ownership-safe fail-closed cleanup', () => {
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ const secretResolved = firstPatchWithType(patchedConditionSets(customApi), 'SecretResolved')
+ expect(secretResolved).toBeDefined()
+ expect(secretResolved?.some(condition => condition.reason === 'ReadError')).toBe(true)
+ expect(secretResolved?.map(condition => condition.type)).not.toContain('NetworkReady')
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
})
it('skips cleanup when the existing runtime is WRC-owned', async () => {
@@ -1430,17 +2121,184 @@ describe('ownership-safe fail-closed cleanup', () => {
expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedConfigMap).not.toHaveBeenCalled()
expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'SecretResolved' && condition.reason === 'SecretNotFound'
+ )
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'Ready' && condition.reason === 'SecretValidationFailed'
+ )
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)
+ })
+
+ it('keeps the obligation when a foreign Deployment sits next to a denied HCC Service', async () => {
+ const missing = new Error('missing') as Error & { code?: number }
+ missing.code = 404
+ const notFound = new Error('not found') as Error & { code?: number }
+ notFound.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(missing)
+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({
+ metadata: {
+ labels: {
+ [MANAGED_BY_LABEL]: WRC_MANAGED_BY_VALUE,
+ [MCPSERVER_LABEL]: 'pg',
+ },
+ },
+ })
+ coreApi.readNamespacedConfigMap.mockRejectedValueOnce(notFound)
+ let seenServiceRead = false
+ let deniedServiceDelete = false
+ coreApi.readNamespacedService.mockImplementation(async () => {
+ seenServiceRead = true
+ return { metadata: hccOwnedMetadata('pg') }
+ })
+
+ const server = makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ await reconciler.reconcile(server, {
+ isCurrent: () => {
+ if (seenServiceRead && !deniedServiceDelete) {
+ deniedServiceDelete = true
+ return false
+ }
+ return true
+ },
+ })
+
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('disabled managed server still writes Disabled when the runtime is WRC-owned', async () => {
+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({
+ metadata: {
+ labels: {
+ [MANAGED_BY_LABEL]: WRC_MANAGED_BY_VALUE,
+ [MCPSERVER_LABEL]: 'pg',
+ },
+ },
+ })
+
+ await reconciler.reconcile(makeServer({ name: 'pg', managed: true, enabled: false }))
+
+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(false)
+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)
+ })
+
+ it('fail-closed status write failure after Deployment delete keeps obligation and publishes not-ready', async () => {
+ const err = new Error('not found') as Error & { code?: number }
+ err.code = 404
+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+ failRuntimeNotDesiredWrites(customApi)
+
+ const server = persistReadySeed(
+ makeServer({
+ name: 'pg',
+ managed: true,
+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },
+ })
+ )
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'Ready' && condition.reason === 'SecretValidationFailed'
+ )
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'SecretResolved' && condition.status === 'False'
+ )
+ )
+ ).toBe(true)
+ expect(reconciler.getStatus(server)).toMatchObject({
+ deployed: false,
+ ready: false,
+ authoritative: true,
+ })
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
+ })
+
+ it('disabled status write failure after Deployment delete still publishes Disabled', async () => {
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+ failRuntimeNotDesiredWrites(customApi)
+
+ const server = persistReadySeed(makeServer({ name: 'pg', managed: true, enabled: false }))
+ await reconciler.reconcile(server)
+
+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')
+ )
+ ).toBe(true)
+ expect(
+ patchedConditionSets(customApi).some(conditions =>
+ conditions.some(
+ condition => condition.type === 'SecretResolved' && condition.status === 'False'
+ )
+ )
+ ).toBe(false)
+ expect(reconciler.getStatus(server)).toMatchObject({
+ deployed: false,
+ ready: false,
+ authoritative: true,
+ })
+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)
})
})
describe('full reconciliation inventory authority', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
customApi.getNamespacedCustomObject.mockRejectedValue(
Object.assign(new Error('not found'), { code: 404 })
)
@@ -1936,13 +2794,15 @@ describe('full reconciliation inventory authority', () => {
})
describe('updateStatusConditions', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let reconciler: McpServerReconciler
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
@@ -1951,42 +2811,34 @@ describe('updateStatusConditions', () => {
})
})
- it('preserves unrelated conditions while updating NetworkReady and DeploymentReady', async () => {
+ it('updateStatusConditions removes NetworkReady and preserves SecretResolved and DeploymentReady in one patch', async () => {
const server = makeServer({ name: 'pg' })
- customApi.getNamespacedCustomObjectStatus
- .mockResolvedValueOnce({
- status: {
- conditions: [
- {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'ok',
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- },
- ],
- },
- })
- .mockResolvedValueOnce({
- status: {
- conditions: [
- {
- type: 'SecretResolved',
- status: 'True',
- reason: 'SecretFound',
- message: 'ok',
- lastTransitionTime: '2020-01-01T00:00:00.000Z',
- },
- {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'NetworkPolicies and Service created',
- lastTransitionTime: '2020-01-02T00:00:00.000Z',
- },
- ],
- },
- })
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
+
+ await (
+ reconciler as unknown as {
+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>
+ }
+ ).updateStatusConditions(server, true)
+
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ const finalConditions = patchedConditionSets(customApi)[0]
+ expect(finalConditions.map(condition => condition.type)).toEqual([
+ 'SecretResolved',
+ 'DeploymentReady',
+ ])
+ expect(
+ finalConditions.find(condition => condition.type === 'SecretResolved')?.lastTransitionTime
+ ).toBe(secretResolvedTrue().lastTransitionTime)
+ expect(
+ finalConditions.find(condition => condition.type === 'DeploymentReady')?.lastTransitionTime
+ ).toBe(deploymentReadyTrue().lastTransitionTime)
+ })
+
+ it('preserves unrelated conditions while updating DeploymentReady and removing NetworkReady', async () => {
+ const server = makeServer({ name: 'pg' })
+ installLiveStatus(customApi, [secretResolvedTrue(), networkReadyTrue()])
await (
reconciler as unknown as {
@@ -1994,42 +2846,42 @@ describe('updateStatusConditions', () => {
}
).updateStatusConditions(server, false)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)
- const finalPatch =
- customApi.patchNamespacedCustomObjectStatus.mock.calls[
- customApi.patchNamespacedCustomObjectStatus.mock.calls.length - 1
- ][0]
- const finalOp = finalPatch.body[0]
- const finalConditions = finalOp.path === '/status' ? finalOp.value.conditions : finalOp.value
- const types = finalConditions.map((c: { type: string }) => c.type)
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ const types = patchedConditionSets(customApi)[0].map(condition => condition.type)
expect(types).toContain('SecretResolved')
- expect(types).toContain('NetworkReady')
+ expect(types).not.toContain('NetworkReady')
expect(types).toContain('DeploymentReady')
})
- it('does not log Updated status conditions when both writes were skipped', async () => {
+ it('does not log Updated status conditions when the write was skipped', async () => {
const server = makeServer({ name: 'pg' })
- let liveStatus: { conditions?: unknown[] } | undefined
- customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({
- metadata: { resourceVersion: '1' },
- ...(liveStatus ? { status: liveStatus } : {}),
- }))
- customApi.patchNamespacedCustomObjectStatus.mockImplementation(
- async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {
- const op = req.body.find(
- candidate =>
- candidate.op === 'add' &&
- (candidate.path === '/status' || candidate.path === '/status/conditions')
- )
- if (!op) throw new Error('status condition add operation was not emitted')
- liveStatus =
- op.path === '/status'
- ? (op.value as { conditions?: unknown[] })
- : { conditions: op.value as unknown[] }
- return {}
+ installLiveStatus(customApi, [deploymentReadyTrue()])
+
+ const updateStatusConditions = (
+ reconciler as unknown as {
+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>
}
- )
+ ).updateStatusConditions.bind(reconciler)
+
+ await updateStatusConditions(server, true)
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+
+ customApi.getNamespacedCustomObjectStatus.mockClear()
+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})
+ try {
+ await updateStatusConditions(server, true)
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ expect(log.mock.calls.flat().join('\n')).not.toContain('Updated status conditions')
+ } finally {
+ log.mockRestore()
+ }
+ })
+ it('skips a second identical updateStatusConditions call after NetworkReady was stripped', async () => {
+ const server = makeServer({ name: 'pg' })
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])
const updateStatusConditions = (
reconciler as unknown as {
updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>
@@ -2037,30 +2889,40 @@ describe('updateStatusConditions', () => {
).updateStatusConditions.bind(reconciler)
await updateStatusConditions(server, true)
- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()
- expect(liveStatus?.conditions).toEqual(
- expect.arrayContaining([
- expect.objectContaining({ type: 'NetworkReady' }),
- expect.objectContaining({ type: 'DeploymentReady' }),
- ])
- )
+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
customApi.patchNamespacedCustomObjectStatus.mockClear()
+ customApi.getNamespacedCustomObjectStatus.mockClear()
const log = vi.spyOn(console, 'log').mockImplementation(() => {})
try {
await updateStatusConditions(server, true)
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
expect(log.mock.calls.flat().join('\n')).not.toContain('Updated status conditions')
} finally {
log.mockRestore()
}
})
+
+ it('does not patch when DeploymentReady already matches and NetworkReady is absent', async () => {
+ const server = makeServer({ name: 'pg' })
+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue()])
+
+ await (
+ reconciler as unknown as {
+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>
+ }
+ ).updateStatusConditions(server, true)
+
+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)
+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()
+ })
})
describe('unconfigured inventory authority fails closed (G1)', () => {
- const appsApi = createMockAppsApi()
- const coreApi = createMockCoreApi()
- const customApi = createMockCustomApi()
+ let appsApi: ReturnType<typeof createMockAppsApi>
+ let coreApi: ReturnType<typeof createMockCoreApi>
+ let customApi: ReturnType<typeof createMockCustomApi>
let errorSpy: ReturnType<typeof vi.spyOn>
// Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO
@@ -2074,7 +2936,9 @@ describe('unconfigured inventory authority fails closed (G1)', () => {
}
beforeEach(() => {
- vi.clearAllMocks()
+ appsApi = createMockAppsApi()
+ coreApi = createMockCoreApi()
+ customApi = createMockCustomApi()
errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
})
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index 64cd8ac76..5ce469df1 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -139,6 +139,50 @@ const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10
// Status is a read/merge/write transaction shared with other controllers.
// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.
const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3
+/** Types HCC no longer writes and strips from every status write (#606). A future writer must not reuse `NetworkReady`; remove the entry first. */
+const RETIRED_STATUS_CONDITION_TYPES = ['NetworkReady'] as const
+export const RUNTIME_NOT_DESIRED_DISABLED_MESSAGE =
+ 'McpServer is disabled; HCC does not run its runtime'
+export const RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE =
+ 'Env Secret validation failed; HCC does not run its runtime'
+
+function isRetiredStatusConditionType(type: string): boolean {
+ return (RETIRED_STATUS_CONDITION_TYPES as readonly string[]).includes(type)
+}
+
+type StatusWriteDecline = 'record-obligation' | 'silent'
+type StatusWriteDeclineArg = () => StatusWriteDecline
+type StatusWriteOutcome = 'wrote' | 'unchanged' | 'declined'
+type RuntimeWithdrawKind = 'disabled' | 'fail-closed'
+type DeploymentDeleteOutcome = 'gone' | 'denied' | 'foreign'
+
+type RetractManagedRuntimeResult = {
+ retracted: boolean
+ cleanupComplete: boolean
+ foreignRuntime: boolean
+ runtimeGone: boolean
+}
+
+function shouldStopAfterIncompleteRetract(
+ retract: RetractManagedRuntimeResult,
+ isCurrent: () => boolean
+): boolean {
+ if (retract.retracted || retract.foreignRuntime) return false
+ return !(retract.runtimeGone && isCurrent())
+}
+
+function runtimeNotDesiredMessage(kind: RuntimeWithdrawKind): string {
+ switch (kind) {
+ case 'disabled':
+ return RUNTIME_NOT_DESIRED_DISABLED_MESSAGE
+ case 'fail-closed':
+ return RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE
+ default: {
+ const _exhaustive: never = kind
+ return _exhaustive
+ }
+ }
+}
function throwCleanupFailures(failures: unknown[], message: string): void {
if (failures.length > 0) {
@@ -257,6 +301,12 @@ interface ReadinessPollState {
*/
isCurrent: () => boolean
server: McpServerCRD
+ /**
+ * Snapshot of `readinessRetirements` at window create time (#606). A later
+ * retirement increments the name's counter; the tick must not publish after
+ * that, even if this state was already deleted from `readinessPolls`.
+ */
+ retirementEpoch: number
}
export class McpServerReconciler {
@@ -302,6 +352,13 @@ export class McpServerReconciler {
* sticky for the same Deployment generation.
*/
private readonly readinessPolls: Map<string, ReadinessPollState> = new Map()
+ /**
+ * Retirement epoch per server name (#606). Incremented when a disabled or
+ * fail-closed reconcile retracts the readiness window. `clearStatus` does
+ * not delete entries: a recreated name keeps the counter so stale ticks fail
+ * the epoch check in addition to the uid fence.
+ */
+ private readonly readinessRetirements = new Map<string, number>()
/**
* Tail promise of the in-flight reconcile chain per server name. Serializes
@@ -348,6 +405,26 @@ export class McpServerReconciler {
}
}
+ /**
+ * Retract the live readiness window for a name that will no longer run a
+ * runtime (#606). Increments the retirement epoch first so an in-flight tick
+ * that already deleted its map entry still fails `retiredNow()`. Clears any
+ * pending timer. Does not record an incomplete-publication obligation. Poll
+ * ticks pass an `onDecline` factory evaluated at decline time: silent only
+ * when `retiredNow()` is true, otherwise the revision fence still records
+ * an obligation.
+ */
+ private retireReadinessWindow(name: string): void {
+ this.readinessRetirements.set(name, (this.readinessRetirements.get(name) ?? 0) + 1)
+ const poll = this.readinessPolls.get(name)
+ if (poll === undefined) return
+ if (poll.timer !== undefined) {
+ clearTimeout(poll.timer)
+ poll.timer = undefined
+ }
+ this.readinessPolls.delete(name)
+ }
+
constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {
this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)
this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)
@@ -618,6 +695,7 @@ export class McpServerReconciler {
detail: previous?.detail ?? '',
isCurrent,
server,
+ retirementEpoch: this.readinessRetirements.get(name) ?? 0,
}
this.readinessPolls.set(name, state)
@@ -627,6 +705,11 @@ export class McpServerReconciler {
// can tell "tick running" apart from "window ended" during our awaits.
state.timer = undefined
state.ticking = true
+ const retiredNow = (): boolean =>
+ (this.readinessRetirements.get(name) ?? 0) !== state.retirementEpoch
+ const publishIfCurrent = (): boolean => state.isCurrent() && !retiredNow()
+ const pollWriteDecline = (): StatusWriteDecline =>
+ retiredNow() ? 'silent' : 'record-obligation'
// Mutation fencing: a superseded reconcile (desired-revision or
// inventory-authority change) must not keep observing — or writing
// status — on behalf of a stale desired state. End the window quietly;
@@ -692,10 +775,18 @@ export class McpServerReconciler {
// always false, so this is the ONLY place a converged rollout is ever
// recorded — without it every successful credential rotation would sit
// at DeploymentReady=False forever and the UI would report a timeout.
- await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)
+ if (retiredNow()) return
+ await this.updateStatusConditions(
+ state.server,
+ true,
+ undefined,
+ publishIfCurrent,
+ pollWriteDecline
+ )
// Readiness is decoupled from fleet convergence: the poll also
// upgrades the persisted, generation-matched Ready condition so a
// controller restart can trust the recorded readiness (see getStatus).
+ if (retiredNow()) return
await this.writeStatusCondition(
state.server,
{
@@ -704,7 +795,8 @@ export class McpServerReconciler {
reason: 'ReconcileSuccess',
message: 'Deployment created',
},
- state.isCurrent
+ publishIfCurrent,
+ pollWriteDecline
)
return
}
@@ -725,7 +817,14 @@ export class McpServerReconciler {
// Give up loudly, on the CRD: a stuck rollout must be visible to the
// operator with the numbers that prove it, never left as a stale
// condition or a silent Unknown.
- await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)
+ if (retiredNow()) return
+ await this.updateStatusConditions(
+ state.server,
+ false,
+ rollout.detail,
+ publishIfCurrent,
+ pollWriteDecline
+ )
return
}
@@ -1657,37 +1756,169 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
name: string,
namespace: string,
deleteAllowed?: () => Promise<boolean>
- ): Promise<void> {
+ ): Promise<DeploymentDeleteOutcome> {
let deployment: k8s.V1Deployment
try {
deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[Reconciler] Deployment "${name}" already gone`)
- } else {
- console.error(`[Reconciler] Failed to read Deployment "${name}" ownership:`, error)
- throw error
+ hccLogger.info('Deployment already gone', { name })
+ return 'gone'
}
- return
+ hccLogger.error('Failed to read Deployment ownership', { name, err: error })
+ throw error
}
if (!this.isHccOwnedMcpResource(deployment, name)) {
- console.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)
- return
+ hccLogger.warn('Skipping Deployment delete — not HCC-owned', { name })
+ return 'foreign'
}
- if (deleteAllowed && !(await deleteAllowed())) return
+ if (deleteAllowed && !(await deleteAllowed())) return 'denied'
try {
await this.appsApi.deleteNamespacedDeployment({ name, namespace })
- console.log(`[Reconciler] Deleted Deployment "${name}"`)
+ hccLogger.info('Deleted Deployment', { name })
+ return 'gone'
} catch (error: unknown) {
if (getErrorCode(error) === 404) {
- console.log(`[Reconciler] Deployment "${name}" already gone`)
- } else {
- console.error(`[Reconciler] Failed to delete Deployment "${name}":`, error)
- throw error
+ hccLogger.info('Deployment already gone', { name })
+ return 'gone'
+ }
+ hccLogger.error('Failed to delete Deployment', { name, err: error })
+ throw error
+ }
+ }
+
+ /**
+ * Delete leftover HCC-owned ConfigMap and Service after the Deployment
+ * outcome is known. Does not write status.
+ */
+ private async deleteHccOwnedRuntimeSiblings(
+ name: string,
+ namespace: string,
+ deleteAllowed?: () => Promise<boolean>
+ ): Promise<void> {
+ const siblingFailures: unknown[] = []
+ for (const cleanup of [
+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
+ ]) {
+ try {
+ await cleanup()
+ } catch (error) {
+ siblingFailures.push(error)
}
}
+ throwCleanupFailures(
+ siblingFailures,
+ `Failed to delete runtime Kubernetes resources for McpServer "${name}"`
+ )
+ }
+
+ /**
+ * Withdraw an HCC-owned runtime (#606): once the Deployment is confirmed
+ * gone, retire the poll, persist DeploymentReady=False/RuntimeNotDesired,
+ * then finish sibling deletes. Sibling failure does not roll back the
+ * retract. Retire before the retract write so an in-flight tick cannot
+ * republish True after the retract PATCH commits but before the epoch
+ * bumps. A foreign (WRC-owned) Deployment retires the window without
+ * writing RuntimeNotDesired.
+ */
+ private async retractManagedRuntime(
+ server: McpServerCRD,
+ kind: RuntimeWithdrawKind,
+ isCurrent: () => boolean
+ ): Promise<RetractManagedRuntimeResult> {
+ const name = server.name
+ const namespace = server.namespace
+ const deleteAllowed = async (): Promise<boolean> => isCurrent()
+ let cleanupComplete = true
+ const gatedDeleteAllowed = async (): Promise<boolean> => {
+ const allowed = await deleteAllowed()
+ if (!allowed) cleanupComplete = false
+ return allowed
+ }
+
+ let deploymentOutcome: DeploymentDeleteOutcome
+ try {
+ deploymentOutcome = await this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed)
+ } catch (error) {
+ const failures: unknown[] = [error]
+ for (const cleanup of [
+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),
+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),
+ ]) {
+ try {
+ await cleanup()
+ } catch (siblingError) {
+ failures.push(siblingError)
+ }
+ }
+ throwCleanupFailures(
+ failures,
+ `Failed to delete runtime Kubernetes resources for McpServer "${name}"`
+ )
+ throw new Error(`Failed to delete runtime Kubernetes resources for McpServer "${name}"`)
+ }
+
+ switch (deploymentOutcome) {
+ case 'denied':
+ return {
+ retracted: false,
+ cleanupComplete: false,
+ foreignRuntime: false,
+ runtimeGone: false,
+ }
+ case 'foreign':
+ this.retireReadinessWindow(name)
+ await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
+ return {
+ retracted: false,
+ cleanupComplete: cleanupComplete && isCurrent(),
+ foreignRuntime: true,
+ runtimeGone: false,
+ }
+ case 'gone':
+ break
+ default: {
+ const _exhaustive: never = deploymentOutcome
+ return _exhaustive
+ }
+ }
+
+ this.retireReadinessWindow(name)
+ const retractWrite = await this.writeStatusCondition(
+ server,
+ {
+ type: 'DeploymentReady',
+ status: 'False',
+ reason: 'RuntimeNotDesired',
+ message: runtimeNotDesiredMessage(kind),
+ },
+ isCurrent
+ )
+ const epoch = this.readinessRetirements.get(name)
+ if (retractWrite === 'declined') {
+ hccLogger.warn('Retract write declined', { name, kind, epoch, outcome: retractWrite })
+ } else {
+ hccLogger.info('Retracted managed runtime', { name, kind, epoch, outcome: retractWrite })
+ }
+
+ await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)
+ if (retractWrite === 'declined') {
+ return {
+ retracted: false,
+ cleanupComplete: false,
+ foreignRuntime: false,
+ runtimeGone: true,
+ }
+ }
+ return {
+ retracted: true,
+ cleanupComplete: cleanupComplete && isCurrent(),
+ foreignRuntime: false,
+ runtimeGone: true,
+ }
}
private async deleteConfigMapIfHccOwned(
@@ -1831,9 +2062,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
}
/**
- * G11: Update status conditions on McpServer CRD.
- * Sets NetworkReady and DeploymentReady conditions so WRC can watch
- * for status changes instead of polling with sleeps.
+ * G11: Update DeploymentReady on the McpServer status subresource.
+ * The condition describes Deployment rollout. WRC does not watch this type;
+ * it reads the `clerum.io/network-ready` annotation.
*/
private async updateStatusConditions(
server: McpServerCRD,
@@ -1844,19 +2075,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* that only know the verdict omit it.
*/
rolloutDetail?: string,
- isCurrent: () => boolean = () => true
+ isCurrent: () => boolean = () => true,
+ onDecline: StatusWriteDeclineArg = () => 'record-obligation'
): Promise<void> {
try {
- const networkWrote = await this.writeStatusCondition(
- server,
- {
- type: 'NetworkReady',
- status: 'True',
- reason: 'NetworkPoliciesApplied',
- message: 'NetworkPolicies and Service created',
- },
- isCurrent
- )
const deploymentWrote = await this.writeStatusCondition(
server,
{
@@ -1873,16 +2095,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
? 'Deployment has ready replicas'
: 'Waiting for pods to become ready',
},
- isCurrent
+ isCurrent,
+ onDecline
)
- if (networkWrote || deploymentWrote) {
- console.log(
- `[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`
- )
+ if (deploymentWrote === 'wrote') {
+ hccLogger.info('Updated status conditions', {
+ name: server.name,
+ deploymentReady,
+ })
}
} catch (error) {
// Status subresource may not exist yet — log but don't fail reconciliation
- console.warn(`[Reconciler] Failed to update status conditions on "${server.name}":`, error)
+ hccLogger.warn('Failed to update status conditions', { name: server.name, err: error })
}
}
@@ -1892,24 +2116,41 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
* Merge semantics:
* - Condition with same `type` and unchanged `status` → keep existing
* `lastTransitionTime`, refresh `reason` / `message`.
+ * - `DeploymentReady` with the same `status` but a new `reason` or
+ * `message` → bump `lastTransitionTime` so recovery after
+ * `RuntimeNotDesired` is visible to #223 consumers.
* - Condition with same `type` but different `status` → bump
* `lastTransitionTime` to now.
* - New type → append with fresh `lastTransitionTime`.
+ * - Any write also strips retired types (NetworkReady). If a retired
+ * type is still present, the patch is forced even when the requested
+ * condition is otherwise unchanged.
*
* Side-effect: updates the `clerum_hcc_mcpserver_missing_secret` gauge for
* `SecretResolved` conditions (1 when False, 0 when True).
*
+ * `onDecline` is a factory evaluated at decline time. The default records
+ * an incomplete-publication obligation. Poll ticks pass a factory that is
+ * `'silent'` only while `retiredNow()` is true.
+ *
+ * Returns `'wrote'` after a successful patch, `'unchanged'` when the
+ * requested condition already matches and no retired type remains, or
+ * `'declined'` when the fence or a write error stops the update.
+ *
* Failure modes:
- * - 404 (CRD deleted mid-reconcile) → log + swallow.
+ * - 404 (CRD deleted mid-reconcile) → log + `decline()`.
* - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.
- * - Other errors → log + swallow (status writes are best-effort).
+ * - Other errors or exhausted retries → log + `decline()`.
*/
async writeStatusCondition(
server: McpServerCRD,
condition: Omit<McpServerCondition, 'lastTransitionTime'>,
- isCurrent: () => boolean = () => true
- ): Promise<boolean> {
- if (!isCurrent()) return this.recordIncompletePublication(server)
+ isCurrent: () => boolean = () => true,
+ onDecline: StatusWriteDeclineArg = () => 'record-obligation'
+ ): Promise<StatusWriteOutcome> {
+ const decline = (): StatusWriteOutcome =>
+ onDecline() === 'silent' ? 'declined' : this.recordIncompletePublication(server)
+ if (!isCurrent()) return decline()
const now = new Date().toISOString()
// Update the missing-secret gauge for SecretResolved conditions.
@@ -1934,7 +2175,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// if anything changed since our read; we then re-read and retry, so no
// writer's condition is lost.
for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {
- if (!isCurrent()) return this.recordIncompletePublication(server)
+ if (!isCurrent()) return decline()
// Re-read on every optimistic-conflict retry. Merging against the
// latest resourceVersion preserves conditions written by peer
// controllers between our read and patch.
@@ -1960,15 +2201,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`
)
- return this.recordIncompletePublication(server)
+ return decline()
}
console.warn(
`[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,
error
)
- return this.recordIncompletePublication(server)
+ return decline()
}
- if (!isCurrent()) return this.recordIncompletePublication(server)
+ if (!isCurrent()) return decline()
const prior = existingConditions.find(c => c.type === condition.type)
const observedGeneration = condition.observedGeneration ?? server.generation
@@ -1977,7 +2218,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
prior.status === condition.status &&
prior.reason === condition.reason &&
prior.message === condition.message &&
- prior.observedGeneration === observedGeneration
+ prior.observedGeneration === observedGeneration &&
+ !existingConditions.some(existing => isRetiredStatusConditionType(existing.type))
// Skip the patch when the condition is identical to what's already on
// the CRD. Without this, every reconcile bumps the McpServer's
@@ -1985,11 +2227,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// reconcile — a tight self-loop that also amplifies NetworkPolicy
// optimistic-lock contention downstream.
if (unchanged) {
- return false
+ return 'unchanged'
}
+ const refreshDeploymentReadyTime =
+ condition.type === 'DeploymentReady' &&
+ prior !== undefined &&
+ (prior.reason !== condition.reason || prior.message !== condition.message)
const lastTransitionTime =
- prior && prior.status === condition.status ? prior.lastTransitionTime : now
+ prior && prior.status === condition.status && !refreshDeploymentReadyTime
+ ? prior.lastTransitionTime
+ : now
const merged: McpServerCondition = {
type: condition.type,
@@ -2000,7 +2248,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
...(observedGeneration !== undefined && { observedGeneration }),
}
- const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]
+ const nextConditions = [
+ ...existingConditions.filter(
+ existing =>
+ existing.type !== condition.type && !isRetiredStatusConditionType(existing.type)
+ ),
+ merged,
+ ]
const writePreconditions: Array<{
op: 'test'
@@ -2039,7 +2293,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// The generated client prefers JSON Patch for this endpoint. Send an
// actual patch document so status writes keep working across client
// upgrades instead of relying on merge-patch object bodies.
- if (!isCurrent()) return this.recordIncompletePublication(server)
+ if (!isCurrent()) return decline()
await this.customApi.patchNamespacedCustomObjectStatus({
group: 'clerum.io',
version: 'v1alpha1',
@@ -2048,14 +2302,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
name: server.name,
body: statusPatch,
})
- return true
+ return 'wrote'
} catch (error) {
const errorCode = getErrorCode(error)
if (errorCode === 404) {
console.warn(
`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`
)
- return this.recordIncompletePublication(server)
+ return decline()
}
// 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer
// won the race. Re-read and retry rather than lose our condition. Any
@@ -2071,18 +2325,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
`[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,
error
)
- return this.recordIncompletePublication(server)
+ return decline()
}
}
- return this.recordIncompletePublication(server)
+ return decline()
}
- private recordIncompletePublication(server: McpServerCRD): false {
- // Keep the changed-boolean API: an unchanged condition is successful but
- // returns false. Only a failed/retired write replaces the obligation, so
- // an enclosing reconcile cannot clear that failure on normal return.
+ private recordIncompletePublication(server: McpServerCRD): 'declined' {
+ // A declined or failed write replaces the obligation so an enclosing
+ // reconcile cannot clear that failure on a later successful sibling write.
this.pendingReconciliations.set(server.name, { namespace: server.namespace })
- return false
+ return 'declined'
}
// ─── Public API ─────────────────────────────────────────────────────
@@ -2155,16 +2408,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
// If disabled, respect ownership before any cleanup. managed:false means
// WRC owns runtime resources; HCC only marks discovery status disabled.
if (server.spec.enabled === false) {
+ let cleanupComplete = true
+ let retract: RetractManagedRuntimeResult | undefined
if (currentManaged) {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`
)
- if (
- !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>
- isCurrent()
- ))
- )
- return false
+ retract = await this.retractManagedRuntime(server, 'disabled', isCurrent)
+ if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
+ cleanupComplete = retract.cleanupComplete
} else {
console.log(
`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`
@@ -2182,7 +2434,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- return isCurrent()
+ if (retract && !retract.retracted && !retract.foreignRuntime) return false
+ return cleanupComplete && isCurrent()
}
if (!currentManaged) {
@@ -2196,13 +2449,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
console.error(
`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`
)
- if (shouldFailClosedForSecretFailure(secretResult.reason)) {
- if (
- !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>
- isCurrent()
- ))
- )
- return false
+ const withdrawRuntime = shouldFailClosedForSecretFailure(secretResult.reason)
+ let cleanupComplete = true
+ let retract: RetractManagedRuntimeResult | undefined
+ if (withdrawRuntime) {
+ retract = await this.retractManagedRuntime(server, 'fail-closed', isCurrent)
+ if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false
+ cleanupComplete = retract.cleanupComplete
} else {
console.warn(
`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`
@@ -2234,7 +2487,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
},
isCurrent
)
- return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()
+ if (retract && !retract.retracted && !retract.foreignRuntime) return false
+ return withdrawRuntime && cleanupComplete && isCurrent()
}
// Ensure resources exist and are up to date
@@ -2344,9 +2598,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
await this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)
// PR-B B1: record SecretResolved=True + Ready=True after a successful
- // reconcile. This is additive to updateStatusConditions (NetworkReady /
- // DeploymentReady); the merge logic in writeStatusCondition preserves
- // the existing conditions by type.
+ // reconcile. This is additive to updateStatusConditions (DeploymentReady);
+ // the merge logic in writeStatusCondition preserves the existing
+ // conditions by type.
if (server.spec.envSecret) {
await this.writeStatusCondition(
server,
diff --git a/scripts/e2e/run-vitest-e2e.sh b/scripts/e2e/run-vitest-e2e.sh
index 295a92ae0..c4ebbeb70 100755
--- a/scripts/e2e/run-vitest-e2e.sh
+++ b/scripts/e2e/run-vitest-e2e.sh
@@ -34,6 +34,7 @@ DEFAULT_CLUSTER_VITEST_SUITES=(
integration/mcp-secret-rotation-isolation.test.ts
integration/mcp-secret-rotation-shared.test.ts
integration/mcp-secret-rotation-remote.test.ts
+ integration/mcp-runtime-conditions.test.ts
integration/mcp-hcc-upgrade-rollout.test.ts
integration/mcp-server-readiness-regression.test.ts
integration/mcp-secrets-access-guard.test.ts
diff --git a/tests/e2e/integration/mcp-runtime-conditions.test.ts b/tests/e2e/integration/mcp-runtime-conditions.test.ts
new file mode 100644
index 000000000..d5dd8a73f
--- /dev/null
+++ b/tests/e2e/integration/mcp-runtime-conditions.test.ts
@@ -0,0 +1,190 @@
+/**
+ * E2E: HCC runtime conditions (#606).
+ *
+ * Proves, on a live cluster, that HCC retracts DeploymentReady when it does
+ * not run a managed server and never writes a NetworkReady condition.
+ *
+ * Actions go through control-api (DELETE + POST of the Secret; full-spec PUT
+ * to disable). Do not write the Secret with kubectl after setup, and do not
+ * merge-patch it — PUT is merge-patch and cannot produce a missing key.
+ *
+ * Run:
+ * bash scripts/e2e/run-vitest-e2e.sh integration/mcp-runtime-conditions.test.ts
+ */
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import {
+ MCP_SERVERS_NAMESPACE,
+ ROLLOUT_TIMEOUT_MS,
+ adminLogin,
+ conditionTransitionMs,
+ deleteMcpSecret,
+ deleteMcpServerFixture,
+ getMcpServerResource,
+ kubectlApply,
+ localMcpServerYaml,
+ postMcpSecret,
+ putMcpServer,
+ randomSuffix,
+ requireControlApiUp,
+ requireDeploymentAbsent,
+ requireHccDevModeUnset,
+ secretYaml,
+ waitForRolloutCondition,
+ waitForStatusCondition,
+} from './mcpCredentialRotation.helpers.js'
+
+const RUN_ID = randomSuffix()
+const SERVER_NAME = `e2e-runtime-cond-${RUN_ID}`
+const SECRET_NAME = `${SERVER_NAME}-credentials`
+const SECRET_KEY = 'api-key'
+const INITIAL_SECRET_VALUE = 'initial-valid-value'
+const RESTORED_SECRET_VALUE = 'restored-valid-value'
+
+let controlApiUp = false
+let adminToken = ''
+
+function expectNoNetworkReady(resource: {
+ status?: { conditions?: Array<{ type: string }> }
+}): void {
+ const conditions = resource.status?.conditions
+ expect(conditions, 'status.conditions must be present').toBeDefined()
+ const types = (conditions ?? []).map(condition => condition.type)
+ expect(types, `DeploymentReady must be present; saw ${types.join(',')}`).toContain(
+ 'DeploymentReady'
+ )
+ expect(types, `NetworkReady must be absent; saw ${types.join(',')}`).not.toContain('NetworkReady')
+}
+
+beforeAll(async () => {
+ controlApiUp = await requireControlApiUp('mcp-runtime-conditions')
+ if (!controlApiUp) return
+
+ requireHccDevModeUnset()
+ adminToken = await adminLogin()
+
+ kubectlApply(
+ secretYaml(SECRET_NAME, MCP_SERVERS_NAMESPACE, {
+ [SECRET_KEY]: INITIAL_SECRET_VALUE,
+ })
+ )
+ kubectlApply(
+ localMcpServerYaml({
+ name: SERVER_NAME,
+ envSecretName: SECRET_NAME,
+ keys: [{ secretKey: SECRET_KEY, envVar: 'E2E_ROTATION_API_KEY' }],
+ })
+ )
+}, ROLLOUT_TIMEOUT_MS + 30_000)
+
+afterAll(() => {
+ if (!controlApiUp) return
+ deleteMcpServerFixture(SERVER_NAME, SECRET_NAME)
+})
+
+describe('mcp runtime conditions (#606)', () => {
+ it(
+ 'retracts DeploymentReady on Secret delete, restores it on recreate, and disables the runtime',
+ async () => {
+ if (!controlApiUp) return
+
+ const ready = await waitForRolloutCondition(SERVER_NAME, adminToken, {
+ expectStatus: 'True',
+ expectReason: 'ReplicasAvailable',
+ sinceMs: 0,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))
+
+ const deletedAt = conditionTransitionMs(ready)
+ const deleted = await deleteMcpSecret(SECRET_NAME, adminToken)
+ expect(deleted.status, `DELETE secret failed: ${JSON.stringify(deleted.data)}`).toBe(200)
+
+ await waitForStatusCondition(SERVER_NAME, adminToken, {
+ type: 'SecretResolved',
+ expectStatus: 'False',
+ expectReason: 'SecretNotFound',
+ sinceMs: deletedAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ await waitForStatusCondition(SERVER_NAME, adminToken, {
+ type: 'Ready',
+ expectStatus: 'False',
+ expectReason: 'SecretValidationFailed',
+ sinceMs: deletedAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ const retired = await waitForRolloutCondition(SERVER_NAME, adminToken, {
+ expectStatus: 'False',
+ expectReason: 'RuntimeNotDesired',
+ sinceMs: deletedAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ expect(retired.message).toContain('Env Secret validation failed')
+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))
+ requireDeploymentAbsent(SERVER_NAME, MCP_SERVERS_NAMESPACE)
+
+ const restoredAt = conditionTransitionMs(retired)
+ const created = await postMcpSecret(
+ SECRET_NAME,
+ { [SECRET_KEY]: RESTORED_SECRET_VALUE },
+ adminToken
+ )
+ expect(created.status, `POST secret failed: ${JSON.stringify(created.data)}`).toBe(201)
+
+ const restored = await waitForRolloutCondition(SERVER_NAME, adminToken, {
+ expectStatus: 'True',
+ expectReason: 'ReplicasAvailable',
+ sinceMs: restoredAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ expect(Date.parse(restored.lastTransitionTime)).toBeGreaterThan(
+ Date.parse(retired.lastTransitionTime)
+ )
+ const recoveredReady = await waitForStatusCondition(SERVER_NAME, adminToken, {
+ type: 'Ready',
+ expectStatus: 'True',
+ expectReason: 'ReconcileSuccess',
+ sinceMs: restoredAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ await waitForStatusCondition(SERVER_NAME, adminToken, {
+ type: 'SecretResolved',
+ expectStatus: 'True',
+ expectReason: 'SecretFound',
+ sinceMs: restoredAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))
+
+ const current = await getMcpServerResource(SERVER_NAME, adminToken)
+ const disabledAt = conditionTransitionMs(recoveredReady)
+ const disabled = await putMcpServer(
+ SERVER_NAME,
+ {
+ metadata: { resourceVersion: current.metadata.resourceVersion },
+ spec: { ...current.spec, enabled: false },
+ },
+ adminToken
+ )
+ expect(disabled.status, `PUT disable failed: ${JSON.stringify(disabled.data)}`).toBe(200)
+
+ await waitForStatusCondition(SERVER_NAME, adminToken, {
+ type: 'Ready',
+ expectStatus: 'False',
+ expectReason: 'Disabled',
+ sinceMs: disabledAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ const disabledReady = await waitForRolloutCondition(SERVER_NAME, adminToken, {
+ expectStatus: 'False',
+ expectReason: 'RuntimeNotDesired',
+ sinceMs: disabledAt,
+ timeoutMs: ROLLOUT_TIMEOUT_MS,
+ })
+ expect(disabledReady.message).toContain('McpServer is disabled')
+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))
+ requireDeploymentAbsent(SERVER_NAME, MCP_SERVERS_NAMESPACE)
+ },
+ ROLLOUT_TIMEOUT_MS * 8 + 30_000
+ )
+})
diff --git a/tests/e2e/integration/mcpCredentialRotation.helpers.ts b/tests/e2e/integration/mcpCredentialRotation.helpers.ts
index e2bfa9b71..6627640d3 100644
--- a/tests/e2e/integration/mcpCredentialRotation.helpers.ts
+++ b/tests/e2e/integration/mcpCredentialRotation.helpers.ts
@@ -323,6 +323,70 @@ export async function putMcpSecret(
)
}
+export async function deleteMcpSecret(
+ name: string,
+ session: string
+): Promise<{ status: number; data: unknown }> {
+ return deleteJson(
+ `${CONTROL_API_URL}/api/v1/admin/mcp-secrets/${encodeURIComponent(name)}`,
+ adminSessionHeader(session)
+ )
+}
+
+export async function postMcpSecret(
+ name: string,
+ data: Record<string, string>,
+ session: string
+): Promise<{ status: number; data: unknown }> {
+ return postJson(
+ `${CONTROL_API_URL}/api/v1/admin/mcp-secrets`,
+ { name, data },
+ adminSessionHeader(session)
+ )
+}
+
+export async function putMcpServer(
+ name: string,
+ body: { metadata?: { resourceVersion?: string }; spec: Record<string, unknown> },
+ session: string
+): Promise<{ status: number; data: unknown }> {
+ return putJson(
+ `${CONTROL_API_URL}/api/v1/admin/mcp-servers/${encodeURIComponent(name)}`,
+ body,
+ adminSessionHeader(session)
+ )
+}
+
+/** Fail loud when HCC is running with the Secret informer disabled. */
+export function requireHccDevModeUnset(): void {
+ const raw = kubectl(
+ `-n control-plane get deploy host-context-controller -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CLERUM_DEV_MODE")].value}'`
+ )
+ // Match host-context-controller/src/config.ts getEnvBool: true/TRUE/1.
+ if (raw.toLowerCase() === 'true' || raw === '1') {
+ throw new Error(
+ 'HCC pod has CLERUM_DEV_MODE enabled; the Secret informer is off. Stop and fix the profile before running runtime-condition e2e.'
+ )
+ }
+}
+
+/** Milliseconds of a persisted condition timestamp. Throws if unparseable. */
+export function conditionTransitionMs(condition: { lastTransitionTime: string }): number {
+ const parsed = Date.parse(condition.lastTransitionTime)
+ if (Number.isNaN(parsed)) {
+ throw new Error(`invalid lastTransitionTime: ${condition.lastTransitionTime}`)
+ }
+ return parsed
+}
+
+/** Empty name means the Deployment is gone. Any kubectl error throws. */
+export function requireDeploymentAbsent(name: string, namespace: string): void {
+ const found = kubectl(`get deploy ${name} -n ${namespace} --ignore-not-found -o name`)
+ if (found !== '') {
+ throw new Error(`expected Deployment "${name}" to be absent, found ${found}`)
+ }
+}
+
export async function getMcpServerResource(
name: string,
session: string
@@ -346,29 +410,39 @@ export function deploymentReadyCondition(
}
/**
- * Poll GET /admin/mcp-servers/:name until DeploymentReady reaches
- * `expectStatus` with a `lastTransitionTime` strictly after `sinceMs`.
+ * Poll GET /admin/mcp-servers/:name until a status condition reaches
+ * `expectStatus` (and optional `expectReason`) with a `lastTransitionTime`
+ * at or after `sinceMs`.
*
- * The `sinceMs` correlation is load-bearing (plan Fase 3, requisito 6 /
- * hallazgo 1): without it, a condition left over from a PREVIOUS deploy would
- * satisfy the predicate instantly, before the rollout under test even began.
+ * The `sinceMs` correlation is load-bearing: without it, a leftover
+ * condition from a previous deploy would satisfy the predicate instantly,
+ * before the rollout under test even began. Prefer a previous condition's
+ * server-side timestamp over the host clock so VM clock skew cannot flake.
*/
-export async function waitForRolloutCondition(
+export async function waitForStatusCondition(
name: string,
token: string,
- opts: { expectStatus: 'True' | 'False'; sinceMs: number; timeoutMs: number }
+ opts: {
+ type: string
+ expectStatus: 'True' | 'False'
+ expectReason?: string
+ sinceMs: number
+ timeoutMs: number
+ }
): Promise<McpServerCondition> {
+ const reasonLabel = opts.expectReason ? ` reason=${opts.expectReason}` : ''
return waitFor(
- `McpServer "${name}" DeploymentReady=${opts.expectStatus} with lastTransitionTime after ${new Date(
+ `McpServer "${name}" ${opts.type}=${opts.expectStatus}${reasonLabel} with lastTransitionTime at or after ${new Date(
opts.sinceMs
).toISOString()}`,
async () => {
const resource = await getMcpServerResource(name, token)
- const cond = deploymentReadyCondition(resource)
+ const cond = resource.status?.conditions?.find(c => c.type === opts.type)
if (!cond) return null
const transitionMs = Date.parse(cond.lastTransitionTime)
if (Number.isNaN(transitionMs) || transitionMs < opts.sinceMs) return null
if (cond.status !== opts.expectStatus) return null
+ if (opts.expectReason !== undefined && cond.reason !== opts.expectReason) return null
return cond
},
opts.timeoutMs,
@@ -382,6 +456,27 @@ export async function waitForRolloutCondition(
)
}
+/**
+ * Poll GET /admin/mcp-servers/:name until DeploymentReady reaches
+ * `expectStatus` with a `lastTransitionTime` at or after `sinceMs`.
+ *
+ * The `sinceMs` correlation is load-bearing: without it, a leftover
+ * condition from a previous deploy would satisfy the predicate instantly,
+ * before the rollout under test even began.
+ */
+export async function waitForRolloutCondition(
+ name: string,
+ token: string,
+ opts: {
+ expectStatus: 'True' | 'False'
+ expectReason?: string
+ sinceMs: number
+ timeoutMs: number
+ }
+): Promise<McpServerCondition> {
+ return waitForStatusCondition(name, token, { type: 'DeploymentReady', ...opts })
+}
+
/**
* E2's guardian: poll until the rollout reaches a TERMINAL failure
* (`DeploymentReady=False`, `reason=RolloutIncomplete`) after `sinceMs`. If a
diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json
index a50f0aec9..7d7aecf0b 100644
--- a/tests/e2e/package-lock.json
+++ b/tests/e2e/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.80",
+ "version": "0.1.83",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-e2e-tests",
- "version": "0.1.80",
+ "version": "0.1.83",
"devDependencies": {
"@types/jsonwebtoken": "^9.0.10",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/package.json b/tests/e2e/package.json
index 5d00434de..aad2a7b47 100644
--- a/tests/e2e/package.json
+++ b/tests/e2e/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-e2e-tests",
- "version": "0.1.80",
+ "version": "0.1.83",
"private": true,
"type": "module",
"scripts": {