Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): retract runtime conditions when HCC does not run a server (#633)
ALFREDO J LOPEZ H (alfredolopez80.eth)2502a3cfe7d3
diff --git a/docs/architecture/overview.md b/docs/architecture/overview.mdindex b4ef088c3..1aa757d23 100644--- a/docs/architecture/overview.md+++ b/docs/architecture/overview.md@@ -898,7 +898,7 @@ flowchart LRDEP --> SVC["3. Create ClusterIP Service"]SVC --> CONFLICT["4. 409? preserve resourceVersion,<br/>replace"]CONFLICT --> SIDECAR["5. stdio-bridge sidecar<br/>(managed:true + stdio transport)"]- SIDECAR --> STATUS["6. Patch status:<br/>NetworkReady, DeploymentReady"]+ SIDECAR --> STATUS["6. Patch status:<br/>DeploymentReady<br/>(RuntimeNotDesired when HCC does not run the runtime)"]endsubgraph L2["Context Watch Loop"]CTX["Context CRD<br/>created/modified"] --> NP["Generate NetworkPolicies<br/>per (context, server) pair"]@@ -933,7 +933,7 @@ When an McpServer CRD is created or modified:- **MCP Proxy** routes HTTP requests to the sidecar transparently- Security overrides (`addCapabilities`, `runAsUser`, etc.) are propagated from the McpServer CRD-6. **Status conditions** (G11): After reconciliation, HCC patches McpServer CRD status with `NetworkReady` and `DeploymentReady` conditions for reactive watches.+6. **Status conditions** (G11): After reconciliation, HCC patches McpServer CRD status with `DeploymentReady` for the Deployment rollout. When HCC does not run a **managed** runtime (disabled or fail-closed secret validation) **and the HCC-owned Deployment is confirmed gone**, it writes `DeploymentReady=False/RuntimeNotDesired`. `McpServerReconciler.writeStatusCondition` strips leftover `NetworkReady` from every McpServer status write; `NetworkPolicyReconciler` may still echo a leftover `NetworkReady` until the next McpServer write removes it. WRC watches the `clerum.io/network-ready` annotation, not a `NetworkReady` condition.7. **Orphan cleanup**: On startup, full reconciliation deletes any Deployments with `clerum.io/managed-by=host-context-controller` that no longer have a corresponding McpServer CRD.diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 5e911645c..3b1d90381 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.322",+ "version": "0.1.329","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.322",+ "version": "0.1.329","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 3d9e0bf25..514f30a40 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.322",+ "version": "0.1.329","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.tsindex ce5e7df25..96ba5134c 100644--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts@@ -8,7 +8,7 @@ import {createMockCoreApi,createMockCustomApi,} from '../../test/__fixtures__/testMocks'-import { McpServerReconciler } from '../reconciler'+import { McpServerReconciler, RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE } from '../reconciler'import { McpServerCRD } from '../types'vi.mock('../config', () => ({@@ -72,6 +72,99 @@ function credentialsRevisionOf(deployment: k8s.V1Deployment): string | undefinedreturn deployment.spec?.template?.metadata?.annotations?.[CREDENTIALS_REVISION_ANNOTATION]}+function readinessPollsOf(reconciler: McpServerReconciler): Map<string, unknown> {+ return (reconciler as unknown as { readinessPolls: Map<string, unknown> }).readinessPolls+}++function notConvergedDeployment(generation: number, name = 'linear') {+ return {+ metadata: {+ name,+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },+ spec: { replicas: 1 },+ status: {+ observedGeneration: generation,+ replicas: 2,+ updatedReplicas: 1,+ readyReplicas: 1,+ availableReplicas: 1,+ unavailableReplicas: 0,+ },+ }+}++function holdNextStatusRead(customApi: ReturnType<typeof createMockCustomApi>): {+ release: (value?: unknown) => void+ resolved: () => number+ isHeld: () => boolean+} {+ let pending: ((value: unknown) => void) | undefined+ let resolved = 0+ let holdNext = true+ customApi.getNamespacedCustomObjectStatus.mockImplementation(+ () =>+ new Promise(resolve => {+ if (holdNext) {+ holdNext = false+ pending = value => {+ resolved += 1+ resolve(value)+ }+ return+ }+ resolve({ metadata: { resourceVersion: '1' }, status: { conditions: [] } })+ })+ )+ return {+ isHeld: () => pending !== undefined,+ resolved: () => resolved,+ release: (value = { metadata: { resourceVersion: '1' }, status: { conditions: [] } }) => {+ if (pending === undefined) {+ throw new Error('status GET was never held')+ }+ pending(value)+ },+ }+}++function laterConditionWrites(+ customApi: ReturnType<typeof createMockCustomApi>,+ afterCallCount: number+): Array<{ type: string; status: string; reason: string }> {+ return customApi.patchNamespacedCustomObjectStatus.mock.calls+ .slice(afterCallCount)+ .flatMap(call => {+ const body = (call[0] as { body?: unknown }).body+ return Array.isArray(body) ? body : []+ })+ .flatMap((op: unknown) => {+ const value = (op as { value?: unknown }).value+ const conditions = Array.isArray(value)+ ? value+ : ((value as { conditions?: unknown[] })?.conditions ?? [])+ return Array.isArray(conditions) ? conditions : []+ })+ .map((c: unknown) => c as { type: string; status: string; reason: string })+}++async function reconcileFailClosed(+ reconciler: McpServerReconciler,+ coreApi: ReturnType<typeof createMockCoreApi>+) {+ coreApi.readNamespacedSecret.mockRejectedValue(+ Object.assign(new Error('not found'), { code: 404 })+ )+ await reconciler.reconcile(makeServer())+}+/*** Every DeploymentReady condition written to the CRD, oldest first. Conditions* travel as a JSON Patch on the status subresource, so this digs the condition@@ -122,13 +215,15 @@ function convergedDeployment(name = 'linear') {}describe('connector credentials revision (issue #223)', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()let live: k8s.V1Deployment | undefined// This suite creates a connector, then observes the same API object on rotation.appsApi.readNamespacedDeployment.mockImplementation(async () => {@@ -707,6 +802,252 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',expect(writes.at(-1)?.status).toBe('True')expect(writes.some(w => w.reason === 'RolloutIncomplete')).toBe(false)})++ it('open readiness window does not write RolloutIncomplete after fail-closed retirement', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer())+ const readsBeforeTick = appsApi.readNamespacedDeployment.mock.calls.length+ await vi.advanceTimersByTimeAsync(5000)+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBeGreaterThan(readsBeforeTick)++ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ try {+ await reconcileFailClosed(reconciler, coreApi)+ const lastAfterRetirement = deploymentReadyWrites(customApi).at(-1)+ expect(lastAfterRetirement?.reason).toBe('RuntimeNotDesired')+ expect(lastAfterRetirement?.message).toBe(RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE)+ const warnsAfterRetirement = warn.mock.calls.length+ const writesAfterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length++ await vi.advanceTimersByTimeAsync(24 * 5000)++ expect(+ laterConditionWrites(customApi, writesAfterRetirement).some(+ condition =>+ condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'+ )+ ).toBe(false)+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')+ expect(warn.mock.calls.slice(warnsAfterRetirement).flat().join('\n')).not.toContain(+ 'not ready after'+ )+ } finally {+ warn.mockRestore()+ }+ })++ it('in-flight ready-branch tick past map delete cannot write True after retirement', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer())++ const held = holdNextStatusRead(customApi)+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)+ expect(held.isHeld()).toBe(true)+ expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()++ await reconcileFailClosed(reconciler, coreApi)+ expect(+ deploymentReadyWrites(customApi).some(+ write =>+ write.reason === 'RuntimeNotDesired' &&+ write.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE+ )+ ).toBe(true)+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length++ held.release()+ await vi.advanceTimersByTimeAsync(0)+ expect(held.resolved()).toBe(1)++ const leaked = laterConditionWrites(customApi, afterRetirement)+ expect(+ leaked.some(+ condition =>+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||+ (condition.type === 'Ready' &&+ condition.status === 'True' &&+ condition.reason === 'ReconcileSuccess')+ )+ ).toBe(false)+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)+ })++ it('in-flight ready tick cannot overwrite RuntimeNotDesired while retract write is in flight', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer())++ const held = holdNextStatusRead(customApi)+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)+ expect(held.isHeld()).toBe(true)++ let releaseRetractPatch: (() => void) | undefined+ let retractPatchHeld!: () => void+ const retractPatchGate = new Promise<void>(resolve => {+ retractPatchHeld = resolve+ })+ const livePatch = customApi.patchNamespacedCustomObjectStatus.getMockImplementation()+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async req => {+ const body = (req as { body?: Array<{ op?: string; path?: string; value?: unknown }> }).body+ const op = body?.find(+ candidate =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ const conditions = (+ Array.isArray(op?.value)+ ? op.value+ : ((op?.value as { conditions?: Array<{ reason?: string }> }).conditions ?? [])+ ) as Array<{ reason?: string }>+ if (conditions.some(condition => condition.reason === 'RuntimeNotDesired')) {+ await new Promise<void>(resolve => {+ releaseRetractPatch = resolve+ retractPatchHeld()+ })+ }+ if (livePatch) return livePatch(req)+ return {}+ })++ const failClosed = reconcileFailClosed(reconciler, coreApi)+ try {+ // Race the held retract PATCH against reconcile return so a missing+ // RuntimeNotDesired write fails here instead of hanging the suite.+ const retractSeen = await Promise.race([+ retractPatchGate.then(() => 'retract' as const),+ failClosed.then(() => 'finished' as const),+ ])+ expect(retractSeen).toBe('retract')+ expect(releaseRetractPatch).toBeDefined()+ const writesWhileRetractHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length++ held.release({ metadata: { resourceVersion: '2' }, status: { conditions: [] } })+ await vi.advanceTimersByTimeAsync(0)++ expect(+ laterConditionWrites(customApi, writesWhileRetractHeld).some(+ condition =>+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||+ (condition.type === 'Ready' &&+ condition.status === 'True' &&+ condition.reason === 'ReconcileSuccess')+ )+ ).toBe(false)++ releaseRetractPatch?.()+ await failClosed+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)+ } finally {+ releaseRetractPatch?.()+ if (livePatch) {+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(livePatch)+ } else {+ customApi.patchNamespacedCustomObjectStatus.mockReset()+ }+ }+ })++ it('in-flight exhaustion tick cannot write RolloutIncomplete after retirement', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer())+ await vi.advanceTimersByTimeAsync(23 * 5000)++ const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)+ try {+ const held = holdNextStatusRead(customApi)+ await vi.advanceTimersByTimeAsync(5000)+ expect(held.isHeld()).toBe(true)+ expect(warn.mock.calls.flat().join('\n')).toContain('not ready after 24 polls')++ await reconcileFailClosed(reconciler, coreApi)+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length+ held.release()+ await vi.advanceTimersByTimeAsync(0)+ expect(held.resolved()).toBe(1)++ expect(+ laterConditionWrites(customApi, afterRetirement).some(+ condition =>+ condition.type === 'DeploymentReady' && condition.reason === 'RolloutIncomplete'+ )+ ).toBe(false)+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')+ } finally {+ warn.mockRestore()+ }+ })++ it('retirement epoch stops a stale ready-branch tick after a later window was armed', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer())+ const windowOne = readinessPollsOf(reconciler).get('linear')+ expect(windowOne).toBeDefined()++ const held = holdNextStatusRead(customApi)+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)+ expect(held.isHeld()).toBe(true)+ expect(readinessPollsOf(reconciler).get('linear')).toBeUndefined()++ coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('rotated-key') } })+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(3))+ await reconciler.reconcile(makeServer())+ const windowTwo = readinessPollsOf(reconciler).get('linear')+ expect(windowTwo).toBeDefined()+ expect(windowTwo).not.toBe(windowOne)++ await reconcileFailClosed(reconciler, coreApi)+ expect(+ deploymentReadyWrites(customApi).some(write => write.reason === 'RuntimeNotDesired')+ ).toBe(true)+ const afterRetirement = customApi.patchNamespacedCustomObjectStatus.mock.calls.length+ expect(readinessPollsOf(reconciler).has('linear')).toBe(false)+ const readsAfterRetract = appsApi.readNamespacedDeployment.mock.calls.length++ held.release()+ await vi.advanceTimersByTimeAsync(24 * 5000)+ expect(held.resolved()).toBe(1)+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBe(readsAfterRetract)++ const leaked = laterConditionWrites(customApi, afterRetirement)+ expect(+ leaked.some(+ condition =>+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||+ (condition.type === 'Ready' && condition.status === 'True')+ )+ ).toBe(false)+ expect(deploymentReadyWrites(customApi).at(-1)?.reason).toBe('RuntimeNotDesired')+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)+ })++ it('in-flight ready tick records obligation when the revision fence flips without retirement', async () => {+ let current = true+ appsApi.readNamespacedDeployment.mockResolvedValue(notConvergedDeployment(2))+ await reconciler.reconcile(makeServer(), { isCurrent: () => current })++ const held = holdNextStatusRead(customApi)+ appsApi.readNamespacedDeployment.mockResolvedValue(convergedDeployment('linear'))+ await vi.advanceTimersByTimeAsync(5000)+ expect(held.isHeld()).toBe(true)+ const writesWhileHeld = customApi.patchNamespacedCustomObjectStatus.mock.calls.length++ current = false+ held.release()+ await vi.advanceTimersByTimeAsync(0)+ expect(held.resolved()).toBe(1)+ expect(+ laterConditionWrites(customApi, writesWhileHeld).some(+ condition =>+ (condition.type === 'DeploymentReady' && condition.status === 'True') ||+ (condition.type === 'Ready' && condition.status === 'True')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })})describe('poll-window fence survives a superseding reconcile (F1, watch-recovery race)', () => {diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.tsindex a14c0a17a..1e1e74d93 100644--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts@@ -169,10 +169,7 @@ function cloneServer(}}-function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {- const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls- const lastCall = calls[calls.length - 1]?.[0]- const patch = lastCall?.body+function conditionsFromStatusPatch(patch: unknown): any[] {const statusAdd = Array.isArray(patch)? patch.find(candidate =>@@ -184,6 +181,17 @@ function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {return Array.isArray(value) ? value : (value?.conditions ?? [])}+function lastPatchedStatusConditions(customApi: MockCustomApi): any[] {+ const calls = customApi.patchNamespacedCustomObjectStatus.mock.calls+ return conditionsFromStatusPatch(calls[calls.length - 1]?.[0]?.body)+}++function allPatchedStatusConditions(customApi: MockCustomApi): any[] {+ return customApi.patchNamespacedCustomObjectStatus.mock.calls.flatMap(call =>+ conditionsFromStatusPatch(call[0]?.body)+ )+}+// ─── Tests ───────────────────────────────────────────────────────────────────describe('McpServerReconciler remote egress proxy', () => {@@ -830,6 +838,52 @@ describe('McpServerReconciler remote egress proxy', () => {expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()expect(coreApi.createNamespacedService).toHaveBeenCalled()})++ it('does not write NetworkReady on the remote reconcile path', async () => {+ coreApi.readNamespacedConfigMap.mockRejectedValueOnce({ code: 404 })+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })+ let liveConditions = [+ {+ type: 'NetworkReady',+ status: 'True',+ reason: 'NetworkPoliciesApplied',+ message: 'NetworkPolicies and Service created',+ lastTransitionTime: '2020-01-03T00:00:00.000Z',+ },+ ]+ customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({+ metadata: { resourceVersion: '1' },+ status: { conditions: liveConditions },+ }))+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(+ async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {+ const op = req.body.find(+ candidate =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ if (op) {+ liveConditions = (+ op.path === '/status'+ ? ((op.value as { conditions?: typeof liveConditions }).conditions ?? [])+ : (op.value as typeof liveConditions)+ ).map(condition => ({ ...condition }))+ }+ return {}+ }+ )+ await reconciler.reconcile(REMOTE_SERVER)++ const allConditions = allPatchedStatusConditions(customApi)+ expect(+ allConditions.some((condition: { type?: string }) => condition.type === 'DeploymentReady')+ ).toBe(true)+ expect(+ allConditions.some((condition: { type?: string }) => condition.type === 'NetworkReady')+ ).toBe(false)+ expect(lastPatchedStatusConditions(customApi).length).toBeGreaterThan(0)+ })})// ─── Test 5: reconcile skips ConfigMap for local servers ───────────────diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 834926800..4e7cdce9d 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -11,7 +11,11 @@ import {createMockNetworkingApi,} from '../test/__fixtures__/testMocks'import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './constants'-import { McpServerReconciler } from './reconciler'+import {+ McpServerReconciler,+ RUNTIME_NOT_DESIRED_DISABLED_MESSAGE,+ RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE,+} from './reconciler'import { McpServerCRD } from './types'describe.each(['Deployment', 'Service'] as const)('read-first %s contract', kind => {@@ -134,14 +138,164 @@ function makeServer(}}+type StatusConditionFixture = {+ type: string+ status: string+ reason: string+ message: string+ lastTransitionTime: string+ observedGeneration?: number+}++/** Secret.data form. Do not write `password: 'literal'` — public-boundary rejects it. */+function storedSecret(value: string): string {+ return Buffer.from(value, 'utf8').toString('base64')+}++function hccOwnedMetadata(name: string) {+ return {+ name,+ namespace: 'mcp-server',+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': name,+ },+ }+}++function networkReadyTrue(): StatusConditionFixture {+ return {+ type: 'NetworkReady',+ status: 'True',+ reason: 'NetworkPoliciesApplied',+ message: 'NetworkPolicies and Service created',+ lastTransitionTime: '2020-01-03T00:00:00.000Z',+ }+}++function deploymentReadyTrue(): StatusConditionFixture {+ return {+ type: 'DeploymentReady',+ status: 'True',+ reason: 'ReplicasAvailable',+ message: 'Deployment has ready replicas',+ lastTransitionTime: '2020-01-02T00:00:00.000Z',+ }+}++function secretResolvedTrue(): StatusConditionFixture {+ return {+ type: 'SecretResolved',+ status: 'True',+ reason: 'SecretFound',+ message: 'ok',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ }+}++function installLiveStatus(+ customApi: ReturnType<typeof createMockCustomApi>,+ initial: StatusConditionFixture[] = []+): { current: () => StatusConditionFixture[] } {+ let conditions = [...initial]+ customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({+ metadata: { resourceVersion: '1' },+ status: { conditions },+ }))+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(+ async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {+ const op = req.body.find(+ candidate =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ if (!op) throw new Error('status condition add operation was not emitted')+ conditions = (+ op.path === '/status'+ ? ((op.value as { conditions?: StatusConditionFixture[] }).conditions ?? [])+ : (op.value as StatusConditionFixture[])+ ).map(condition => ({ ...condition }))+ return {}+ }+ )+ return { current: () => conditions }+}++function persistReadySeed(server: McpServerCRD): McpServerCRD {+ server.uid = server.uid ?? 'uid-pg'+ server.generation = server.generation ?? 3+ server.status = {+ conditions: [+ {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'Deployment created',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ observedGeneration: server.generation,+ },+ ],+ }+ return server+}++function failRuntimeNotDesiredWrites(customApi: ReturnType<typeof createMockCustomApi>): void {+ const livePatch = customApi.patchNamespacedCustomObjectStatus.getMockImplementation()+ customApi.patchNamespacedCustomObjectStatus.mockImplementation(async req => {+ const body = (req as { body: Array<{ op?: string; path?: string; value?: unknown }> }).body+ const op = body.find(+ candidate =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ const conditions = (+ Array.isArray(op?.value)+ ? op.value+ : ((op?.value as { conditions?: Array<{ reason?: string }> }).conditions ?? [])+ ) as Array<{ reason?: string }>+ if (conditions.some(condition => condition.reason === 'RuntimeNotDesired')) {+ throw Object.assign(new Error('status write failed'), { code: 500 })+ }+ if (livePatch) return livePatch(req)+ return {}+ })+}++function patchedConditionSets(+ customApi: ReturnType<typeof createMockCustomApi>+): StatusConditionFixture[][] {+ return customApi.patchNamespacedCustomObjectStatus.mock.calls.map(call => {+ const body = (call[0] as { body: Array<{ op?: string; path?: string; value?: unknown }> }).body+ const op = body.find(+ candidate =>+ candidate.op === 'add' &&+ (candidate.path === '/status' || candidate.path === '/status/conditions')+ )+ if (!op) return []+ const value = op.value+ return (+ Array.isArray(value) ? value : ((value as { conditions?: unknown[] }).conditions ?? [])+ ) as StatusConditionFixture[]+ })+}++function firstPatchWithType(+ sets: StatusConditionFixture[][],+ type: string+): StatusConditionFixture[] | undefined {+ return sets.find(conditions => conditions.some(condition => condition.type === type))+}+describe('Reconciler managed:false guard (Risk 1.7)', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),@@ -479,20 +633,60 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {expect(status.ready).toBe(false)expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.type === 'DeploymentReady')+ )+ ).toBe(false)+ })++ it('does not write DeploymentReady when a WRC-owned server is fail-closed', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ const server = makeServer({+ name: 'workflow-recipes',+ managed: false,+ envSecret: { name: 'missing', keys: [{ secretKey: 'token', envVar: 'TOKEN' }] },+ })++ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'SecretResolved' && condition.reason === 'SecretNotFound'+ )+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.type === 'DeploymentReady')+ )+ ).toBe(false)})})// ─── PR-B B1: validateSecret result shape + writeStatusCondition ───────describe('PR-B B1 — validateSecret result shape', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()- const networkingApi = createMockNetworkingApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>+ let networkingApi: ReturnType<typeof createMockNetworkingApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()+ networkingApi = createMockNetworkingApi()const kubeConfig = new k8s.KubeConfig()// Keep the forbidden API reachable: reintroducing the old optional// NetworkingApi client must not pass by observing an unrelated mock.@@ -654,6 +848,371 @@ describe('PR-B B1 — validateSecret result shape', () => {expect(reconciler.getStatus('pg')).toMatchObject({ deployed: false, ready: false })})+ it('fail-closed incomplete delete does not write RuntimeNotDesired', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ let seenDeploymentRead = false+ let deniedDeploymentDelete = false+ appsApi.readNamespacedDeployment.mockImplementation(async () => {+ seenDeploymentRead = true+ return { metadata: hccOwnedMetadata('pg') }+ })++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ await reconciler.reconcile(server, {+ isCurrent: () => {+ if (seenDeploymentRead && !deniedDeploymentDelete) {+ deniedDeploymentDelete = true+ return false+ }+ return true+ },+ })++ expect(seenDeploymentRead).toBe(true)+ expect(deniedDeploymentDelete).toBe(true)+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('fail-closed writes RuntimeNotDesired when Service cleanup is denied after Deployment delete', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])+ let seenServiceRead = false+ let deniedServiceDelete = false+ coreApi.readNamespacedService.mockImplementation(async () => {+ seenServiceRead = true+ return { metadata: hccOwnedMetadata('pg') }+ })++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ await reconciler.reconcile(server, {+ isCurrent: () => {+ if (seenServiceRead && !deniedServiceDelete) {+ deniedServiceDelete = true+ return false+ }+ return true+ },+ })++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' &&+ condition.reason === 'RuntimeNotDesired' &&+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE+ )+ )+ ).toBe(true)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('disabled writes RuntimeNotDesired when Service cleanup is denied after Deployment delete', async () => {+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])+ let seenServiceRead = false+ let deniedServiceDelete = false+ coreApi.readNamespacedService.mockImplementation(async () => {+ seenServiceRead = true+ return { metadata: hccOwnedMetadata('pg') }+ })++ const server = persistReadySeed(makeServer({ name: 'pg', managed: true, enabled: false }))+ await reconciler.reconcile(server, {+ isCurrent: () => {+ if (seenServiceRead && !deniedServiceDelete) {+ deniedServiceDelete = true+ return false+ }+ return true+ },+ })++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' &&+ condition.reason === 'RuntimeNotDesired' &&+ condition.message === RUNTIME_NOT_DESIRED_DISABLED_MESSAGE+ )+ )+ ).toBe(true)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('fail-closed writes RuntimeNotDesired when Service delete throws after Deployment is gone', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])+ coreApi.deleteNamespacedService.mockRejectedValueOnce(+ Object.assign(new Error('forbidden'), { code: 403 })+ )++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ await expect(reconciler.reconcile(server)).rejects.toThrow(+ 'Failed to delete runtime Kubernetes resources for McpServer "pg"'+ )+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' && condition.reason === 'RuntimeNotDesired'+ )+ )+ ).toBe(true)+ })++ it('disabled incomplete delete does not write RuntimeNotDesired', async () => {+ let seenDeploymentRead = false+ let deniedDeploymentDelete = false+ appsApi.readNamespacedDeployment.mockImplementation(async () => {+ seenDeploymentRead = true+ return { metadata: hccOwnedMetadata('pg') }+ })++ await reconciler.reconcile(makeServer({ name: 'pg', managed: true, enabled: false }), {+ isCurrent: () => {+ if (seenDeploymentRead && !deniedDeploymentDelete) {+ deniedDeploymentDelete = true+ return false+ }+ return true+ },+ })++ expect(deniedDeploymentDelete).toBe(true)+ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('fail-closed SecretNotFound writes RuntimeNotDesired after delete and strips NetworkReady', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })++ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({+ name: 'pg',+ namespace: 'mcp-server',+ })+ const sets = patchedConditionSets(customApi)+ const secretResolvedPatch = firstPatchWithType(sets, 'SecretResolved')+ expect(secretResolvedPatch).toBeDefined()+ expect(secretResolvedPatch?.map(condition => condition.type)).not.toContain('NetworkReady')+ expect(+ sets.some(conditions =>+ conditions.some(c => c.type === 'Ready' && c.reason === 'SecretValidationFailed')+ )+ ).toBe(true)+ const retirementIndex = sets.findIndex(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' &&+ condition.status === 'False' &&+ condition.reason === 'RuntimeNotDesired' &&+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE+ )+ )+ expect(retirementIndex).toBeGreaterThanOrEqual(0)+ expect(sets[retirementIndex]?.map(condition => condition.type)).not.toContain('NetworkReady')+ expect(appsApi.deleteNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(+ customApi.patchNamespacedCustomObjectStatus.mock.invocationCallOrder[retirementIndex]+ )+ })++ it('fail-closed SecretNotFound writes RuntimeNotDesired even when DeploymentReady was absent', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [secretResolvedTrue(), networkReadyTrue()])++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })++ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' &&+ condition.reason === 'RuntimeNotDesired' &&+ condition.message === RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE+ )+ )+ ).toBe(true)+ })++ it('disabled managed server writes RuntimeNotDesired after delete and strips NetworkReady', async () => {+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])++ const server = makeServer({ name: 'pg', managed: true, enabled: false })+ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalledWith({+ name: 'pg',+ namespace: 'mcp-server',+ })+ const sets = patchedConditionSets(customApi)+ expect(+ sets.some(conditions =>+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')+ )+ ).toBe(true)+ const retirementIndex = sets.findIndex(conditions =>+ conditions.some(+ condition =>+ condition.type === 'DeploymentReady' &&+ condition.status === 'False' &&+ condition.reason === 'RuntimeNotDesired' &&+ condition.message === RUNTIME_NOT_DESIRED_DISABLED_MESSAGE+ )+ )+ expect(retirementIndex).toBeGreaterThanOrEqual(0)+ expect(sets.flat().some(condition => condition.type === 'NetworkReady')).toBe(false)+ expect(appsApi.deleteNamespacedDeployment.mock.invocationCallOrder[0]).toBeLessThan(+ customApi.patchNamespacedCustomObjectStatus.mock.invocationCallOrder[retirementIndex]+ )+ })++ it('recovery after fail-closed stamps a fresh WaitingForReplicas time', async () => {+ vi.useFakeTimers()+ try {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [+ secretResolvedTrue(),+ deploymentReadyTrue(),+ networkReadyTrue(),+ ])+ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ await reconciler.reconcile(server)+ const retirement = patchedConditionSets(customApi)+ .flat()+ .find(condition => condition.reason === 'RuntimeNotDesired')+ expect(retirement?.lastTransitionTime).toBeDefined()+ const retiredAt = retirement!.lastTransitionTime++ await vi.advanceTimersByTimeAsync(1000)+ coreApi.readNamespacedSecret.mockResolvedValue({+ data: { password: storedSecret('fixture-key') },+ })+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })+ appsApi.readNamespacedDeployment.mockResolvedValue({+ metadata: {+ name: 'pg',+ namespace: 'mcp-server',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'pg',+ },+ },+ spec: { replicas: 1 },+ status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 0, replicas: 1 },+ })++ await reconciler.reconcile(server)+ const waiting = patchedConditionSets(customApi)+ .flat()+ .filter(condition => condition.type === 'DeploymentReady')+ .at(-1)+ expect(waiting).toMatchObject({+ status: 'False',+ reason: 'WaitingForReplicas',+ })+ expect(Date.parse(waiting!.lastTransitionTime)).toBeGreaterThan(Date.parse(retiredAt))+ expect(+ patchedConditionSets(customApi)+ .flat()+ .some(condition => condition.type === 'NetworkReady')+ ).toBe(false)++ appsApi.readNamespacedDeployment.mockResolvedValue({+ metadata: {+ name: 'pg',+ namespace: 'mcp-server',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'pg',+ },+ },+ spec: { replicas: 1 },+ status: {+ observedGeneration: 2,+ replicas: 1,+ updatedReplicas: 1,+ readyReplicas: 1,+ unavailableReplicas: 0,+ },+ })+ const readsBeforePoll = appsApi.readNamespacedDeployment.mock.calls.length+ await vi.advanceTimersByTimeAsync(5000)+ expect(appsApi.readNamespacedDeployment.mock.calls.length).toBeGreaterThan(readsBeforePoll)+ const ready = patchedConditionSets(customApi)+ .flat()+ .filter(condition => condition.type === 'DeploymentReady')+ .at(-1)+ expect(ready?.status).toBe('True')+ expect(Date.parse(ready!.lastTransitionTime)).toBeGreaterThan(Date.parse(retiredAt))+ expect(+ patchedConditionSets(customApi)+ .flat()+ .some(condition => condition.type === 'NetworkReady')+ ).toBe(false)+ } finally {+ vi.useRealTimers()+ }+ })+it('propagates cleanup errors when secret validation fails', async () => {const err = new Error('not found') as Error & { code?: number }err.code = 404@@ -670,6 +1229,12 @@ describe('PR-B B1 — validateSecret result shape', () => {await expect(reconciler.reconcile(server)).rejects.toThrow('Failed to delete runtime Kubernetes resources for McpServer "pg"')+ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)expect(coreApi.deleteNamespacedService).toHaveBeenCalledWith({name: 'pg',namespace: 'mcp-server',@@ -683,9 +1248,9 @@ describe('PR-B B1 — validateSecret result shape', () => {})describe('PR-B B1 — writeStatusCondition', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerconst getPatchedConditions = () => {@@ -701,7 +1266,9 @@ describe('PR-B B1 — writeStatusCondition', () => {}beforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),@@ -710,6 +1277,69 @@ describe('PR-B B1 — writeStatusCondition', () => {})})+ it('bumps DeploymentReady lastTransitionTime when reason changes at the same status', async () => {+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ status: {+ conditions: [+ {+ type: 'DeploymentReady',+ status: 'False',+ reason: 'RuntimeNotDesired',+ message: RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE,+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ },+ ],+ },+ })++ const server = makeServer({ name: 'pg' })+ await reconciler.writeStatusCondition(server, {+ type: 'DeploymentReady',+ status: 'False',+ reason: 'WaitingForReplicas',+ message: 'Waiting for pods to become ready',+ })++ const { conditions } = getPatchedConditions()+ const written = conditions.find((c: { type: string }) => c.type === 'DeploymentReady')+ expect(written.lastTransitionTime).not.toBe('2020-01-01T00:00:00.000Z')+ expect(Date.parse(written.lastTransitionTime)).toBeGreaterThan(+ Date.parse('2020-01-01T00:00:00.000Z')+ )+ })++ it('bumps DeploymentReady lastTransitionTime when only the message changes', async () => {+ customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({+ status: {+ conditions: [+ {+ type: 'DeploymentReady',+ status: 'False',+ reason: 'WaitingForReplicas',+ message: 'Waiting for pods to become ready',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ },+ ],+ },+ })++ const server = makeServer({ name: 'pg' })+ await reconciler.writeStatusCondition(server, {+ type: 'DeploymentReady',+ status: 'False',+ reason: 'WaitingForReplicas',+ message: 'Rollout did not converge — 0/1 ready',+ })++ const { conditions } = getPatchedConditions()+ const written = conditions.find((c: { type: string }) => c.type === 'DeploymentReady')+ expect(written.lastTransitionTime).not.toBe('2020-01-01T00:00:00.000Z')+ expect(Date.parse(written.lastTransitionTime)).toBeGreaterThan(+ Date.parse('2020-01-01T00:00:00.000Z')+ )+ expect(written.message).toBe('Rollout did not converge — 0/1 ready')+ })+it('preserves lastTransitionTime when status is unchanged', async () => {const existing = {type: 'SecretResolved',@@ -774,9 +1404,9 @@ describe('PR-B B1 — writeStatusCondition', () => {status: {conditions: [{- type: 'NetworkReady',+ type: 'ExternalEgressReady',status: 'True',- reason: 'NetworkPoliciesApplied',+ reason: 'EgressPolicyApplied',message: 'ok',lastTransitionTime: '2020-01-01T00:00:00.000Z',},@@ -794,7 +1424,7 @@ describe('PR-B B1 — writeStatusCondition', () => {const { conditions } = getPatchedConditions()const types = conditions.map((c: { type: string }) => c.type)- expect(types).toContain('NetworkReady')+ expect(types).toContain('ExternalEgressReady')expect(types).toContain('Ready')})@@ -821,28 +1451,28 @@ describe('PR-B B1 — writeStatusCondition', () => {})it('re-reads and preserves a concurrent condition after a resourceVersion conflict', async () => {- const networkReady = {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'network ready',- lastTransitionTime: '2020-01-01T00:00:00.000Z',- }const externalEgressReady = {type: 'ExternalEgressReady',status: 'True',reason: 'EgressPolicyApplied',message: 'external egress ready',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ }+ const peerCondition = {+ type: 'SecretResolved',+ status: 'True',+ reason: 'SecretFound',+ message: 'peer write',lastTransitionTime: '2020-01-02T00:00:00.000Z',}customApi.getNamespacedCustomObjectStatus.mockResolvedValueOnce({metadata: { resourceVersion: '100' },- status: { conditions: [networkReady] },+ status: { conditions: [externalEgressReady] },}).mockResolvedValueOnce({metadata: { resourceVersion: '101' },- status: { conditions: [networkReady, externalEgressReady] },+ status: { conditions: [externalEgressReady, peerCondition] },})customApi.patchNamespacedCustomObjectStatus.mockRejectedValueOnce(Object.assign(new Error('resourceVersion conflict'), { code: 409 }))@@ -878,12 +1508,58 @@ describe('PR-B B1 — writeStatusCondition', () => {candidate.op === 'add' && candidate.path === '/status/conditions')expect(conditionPatch?.value.map((written: { type: string }) => written.type)).toEqual([- 'NetworkReady','ExternalEgressReady',+ 'SecretResolved','Ready',])})+ it('writeStatusCondition of an identical Ready still strips NetworkReady across a 422 retry', async () => {+ const ready = {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ lastTransitionTime: '2020-01-01T00:00:00.000Z',+ }+ const thirdParty = {+ type: 'ExternalEgressReady',+ status: 'True',+ reason: 'EgressPolicyApplied',+ message: 'peer',+ lastTransitionTime: '2020-01-02T00:00:00.000Z',+ }+ customApi.getNamespacedCustomObjectStatus+ .mockResolvedValueOnce({+ metadata: { resourceVersion: '100' },+ status: { conditions: [ready, networkReadyTrue()] },+ })+ .mockResolvedValueOnce({+ metadata: { resourceVersion: '101' },+ status: { conditions: [ready, networkReadyTrue(), thirdParty] },+ })+ customApi.patchNamespacedCustomObjectStatus+ .mockRejectedValueOnce(Object.assign(new Error('conflict'), { code: 422 }))+ .mockResolvedValueOnce({})++ await reconciler.writeStatusCondition(makeServer({ name: 'pg' }), {+ type: 'Ready',+ status: 'True',+ reason: 'ReconcileSuccess',+ message: 'done',+ })++ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)+ const firstTypes = patchedConditionSets(customApi)[0].map(condition => condition.type)+ const secondTypes = patchedConditionSets(customApi)[1].map(condition => condition.type)+ expect(firstTypes).toContain('Ready')+ expect(firstTypes).not.toContain('NetworkReady')+ expect(secondTypes).toContain('Ready')+ expect(secondTypes).toContain('ExternalEgressReady')+ expect(secondTypes).not.toContain('NetworkReady')+ })+it('bounds resourceVersion conflict retries', async () => {const currentStatus = {metadata: { resourceVersion: '100' },@@ -1039,7 +1715,7 @@ describe('PR-B B1 — writeStatusCondition', () => {reason: 'ok',message: 'ok',})- ).resolves.toBe(false)+ ).resolves.toBe('declined')expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()})@@ -1153,13 +1829,15 @@ describe('PR-B B1 — writeStatusCondition', () => {})describe('restart-safe discovery status', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),@@ -1322,13 +2000,15 @@ describe('restart-safe discovery status', () => {})describe('ownership-safe fail-closed cleanup', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),@@ -1388,6 +2068,7 @@ describe('ownership-safe fail-closed cleanup', () => {const err = new Error('api unavailable') as Error & { code?: number }err.code = 500coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [networkReadyTrue()])const server = makeServer({name: 'pg',@@ -1399,6 +2080,16 @@ describe('ownership-safe fail-closed cleanup', () => {expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ const secretResolved = firstPatchWithType(patchedConditionSets(customApi), 'SecretResolved')+ expect(secretResolved).toBeDefined()+ expect(secretResolved?.some(condition => condition.reason === 'ReadError')).toBe(true)+ expect(secretResolved?.map(condition => condition.type)).not.toContain('NetworkReady')+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)})it('skips cleanup when the existing runtime is WRC-owned', async () => {@@ -1430,17 +2121,184 @@ describe('ownership-safe fail-closed cleanup', () => {expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()expect(coreApi.deleteNamespacedConfigMap).not.toHaveBeenCalled()expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'SecretResolved' && condition.reason === 'SecretNotFound'+ )+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'Ready' && condition.reason === 'SecretValidationFailed'+ )+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)+ })++ it('keeps the obligation when a foreign Deployment sits next to a denied HCC Service', async () => {+ const missing = new Error('missing') as Error & { code?: number }+ missing.code = 404+ const notFound = new Error('not found') as Error & { code?: number }+ notFound.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(missing)+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({+ metadata: {+ labels: {+ [MANAGED_BY_LABEL]: WRC_MANAGED_BY_VALUE,+ [MCPSERVER_LABEL]: 'pg',+ },+ },+ })+ coreApi.readNamespacedConfigMap.mockRejectedValueOnce(notFound)+ let seenServiceRead = false+ let deniedServiceDelete = false+ coreApi.readNamespacedService.mockImplementation(async () => {+ seenServiceRead = true+ return { metadata: hccOwnedMetadata('pg') }+ })++ const server = makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ await reconciler.reconcile(server, {+ isCurrent: () => {+ if (seenServiceRead && !deniedServiceDelete) {+ deniedServiceDelete = true+ return false+ }+ return true+ },+ })++ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(coreApi.deleteNamespacedService).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('disabled managed server still writes Disabled when the runtime is WRC-owned', async () => {+ appsApi.readNamespacedDeployment.mockResolvedValueOnce({+ metadata: {+ labels: {+ [MANAGED_BY_LABEL]: WRC_MANAGED_BY_VALUE,+ [MCPSERVER_LABEL]: 'pg',+ },+ },+ })++ await reconciler.reconcile(makeServer({ name: 'pg', managed: true, enabled: false }))++ expect(appsApi.deleteNamespacedDeployment).not.toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(false)+ expect(reconciler.hasIncompleteReconciliation()).toBe(false)+ })++ it('fail-closed status write failure after Deployment delete keeps obligation and publishes not-ready', async () => {+ const err = new Error('not found') as Error & { code?: number }+ err.code = 404+ coreApi.readNamespacedSecret.mockRejectedValueOnce(err)+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])+ failRuntimeNotDesiredWrites(customApi)++ const server = persistReadySeed(+ makeServer({+ name: 'pg',+ managed: true,+ envSecret: { name: 'pg-creds', keys: [{ secretKey: 'password', envVar: 'PGPASSWORD' }] },+ })+ )+ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.reason === 'RuntimeNotDesired')+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'Ready' && condition.reason === 'SecretValidationFailed'+ )+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'SecretResolved' && condition.status === 'False'+ )+ )+ ).toBe(true)+ expect(reconciler.getStatus(server)).toMatchObject({+ deployed: false,+ ready: false,+ authoritative: true,+ })+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)+ })++ it('disabled status write failure after Deployment delete still publishes Disabled', async () => {+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])+ failRuntimeNotDesiredWrites(customApi)++ const server = persistReadySeed(makeServer({ name: 'pg', managed: true, enabled: false }))+ await reconciler.reconcile(server)++ expect(appsApi.deleteNamespacedDeployment).toHaveBeenCalled()+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(condition => condition.type === 'Ready' && condition.reason === 'Disabled')+ )+ ).toBe(true)+ expect(+ patchedConditionSets(customApi).some(conditions =>+ conditions.some(+ condition => condition.type === 'SecretResolved' && condition.status === 'False'+ )+ )+ ).toBe(false)+ expect(reconciler.getStatus(server)).toMatchObject({+ deployed: false,+ ready: false,+ authoritative: true,+ })+ expect(reconciler.hasIncompleteReconciliation()).toBe(true)})})describe('full reconciliation inventory authority', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()customApi.getNamespacedCustomObject.mockRejectedValue(Object.assign(new Error('not found'), { code: 404 }))@@ -1936,13 +2794,15 @@ describe('full reconciliation inventory authority', () => {})describe('updateStatusConditions', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let reconciler: McpServerReconcilerbeforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),@@ -1951,42 +2811,34 @@ describe('updateStatusConditions', () => {})})- it('preserves unrelated conditions while updating NetworkReady and DeploymentReady', async () => {+ it('updateStatusConditions removes NetworkReady and preserves SecretResolved and DeploymentReady in one patch', async () => {const server = makeServer({ name: 'pg' })- customApi.getNamespacedCustomObjectStatus- .mockResolvedValueOnce({- status: {- conditions: [- {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'ok',- lastTransitionTime: '2020-01-01T00:00:00.000Z',- },- ],- },- })- .mockResolvedValueOnce({- status: {- conditions: [- {- type: 'SecretResolved',- status: 'True',- reason: 'SecretFound',- message: 'ok',- lastTransitionTime: '2020-01-01T00:00:00.000Z',- },- {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'NetworkPolicies and Service created',- lastTransitionTime: '2020-01-02T00:00:00.000Z',- },- ],- },- })+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])++ await (+ reconciler as unknown as {+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>+ }+ ).updateStatusConditions(server, true)++ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ const finalConditions = patchedConditionSets(customApi)[0]+ expect(finalConditions.map(condition => condition.type)).toEqual([+ 'SecretResolved',+ 'DeploymentReady',+ ])+ expect(+ finalConditions.find(condition => condition.type === 'SecretResolved')?.lastTransitionTime+ ).toBe(secretResolvedTrue().lastTransitionTime)+ expect(+ finalConditions.find(condition => condition.type === 'DeploymentReady')?.lastTransitionTime+ ).toBe(deploymentReadyTrue().lastTransitionTime)+ })++ it('preserves unrelated conditions while updating DeploymentReady and removing NetworkReady', async () => {+ const server = makeServer({ name: 'pg' })+ installLiveStatus(customApi, [secretResolvedTrue(), networkReadyTrue()])await (reconciler as unknown as {@@ -1994,42 +2846,42 @@ describe('updateStatusConditions', () => {}).updateStatusConditions(server, false)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(2)- const finalPatch =- customApi.patchNamespacedCustomObjectStatus.mock.calls[- customApi.patchNamespacedCustomObjectStatus.mock.calls.length - 1- ][0]- const finalOp = finalPatch.body[0]- const finalConditions = finalOp.path === '/status' ? finalOp.value.conditions : finalOp.value- const types = finalConditions.map((c: { type: string }) => c.type)+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ const types = patchedConditionSets(customApi)[0].map(condition => condition.type)expect(types).toContain('SecretResolved')- expect(types).toContain('NetworkReady')+ expect(types).not.toContain('NetworkReady')expect(types).toContain('DeploymentReady')})- it('does not log Updated status conditions when both writes were skipped', async () => {+ it('does not log Updated status conditions when the write was skipped', async () => {const server = makeServer({ name: 'pg' })- let liveStatus: { conditions?: unknown[] } | undefined- customApi.getNamespacedCustomObjectStatus.mockImplementation(async () => ({- metadata: { resourceVersion: '1' },- ...(liveStatus ? { status: liveStatus } : {}),- }))- customApi.patchNamespacedCustomObjectStatus.mockImplementation(- async (req: { body: Array<{ op?: string; path?: string; value?: unknown }> }) => {- const op = req.body.find(- candidate =>- candidate.op === 'add' &&- (candidate.path === '/status' || candidate.path === '/status/conditions')- )- if (!op) throw new Error('status condition add operation was not emitted')- liveStatus =- op.path === '/status'- ? (op.value as { conditions?: unknown[] })- : { conditions: op.value as unknown[] }- return {}+ installLiveStatus(customApi, [deploymentReadyTrue()])++ const updateStatusConditions = (+ reconciler as unknown as {+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>}- )+ ).updateStatusConditions.bind(reconciler)++ await updateStatusConditions(server, true)+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)++ customApi.getNamespacedCustomObjectStatus.mockClear()+ const log = vi.spyOn(console, 'log').mockImplementation(() => {})+ try {+ await updateStatusConditions(server, true)+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ expect(log.mock.calls.flat().join('\n')).not.toContain('Updated status conditions')+ } finally {+ log.mockRestore()+ }+ })+ it('skips a second identical updateStatusConditions call after NetworkReady was stripped', async () => {+ const server = makeServer({ name: 'pg' })+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue(), networkReadyTrue()])const updateStatusConditions = (reconciler as unknown as {updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>@@ -2037,30 +2889,40 @@ describe('updateStatusConditions', () => {).updateStatusConditions.bind(reconciler)await updateStatusConditions(server, true)- expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalled()- expect(liveStatus?.conditions).toEqual(- expect.arrayContaining([- expect.objectContaining({ type: 'NetworkReady' }),- expect.objectContaining({ type: 'DeploymentReady' }),- ])- )+ expect(customApi.patchNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)customApi.patchNamespacedCustomObjectStatus.mockClear()+ customApi.getNamespacedCustomObjectStatus.mockClear()const log = vi.spyOn(console, 'log').mockImplementation(() => {})try {await updateStatusConditions(server, true)+ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()expect(log.mock.calls.flat().join('\n')).not.toContain('Updated status conditions')} finally {log.mockRestore()}})++ it('does not patch when DeploymentReady already matches and NetworkReady is absent', async () => {+ const server = makeServer({ name: 'pg' })+ installLiveStatus(customApi, [secretResolvedTrue(), deploymentReadyTrue()])++ await (+ reconciler as unknown as {+ updateStatusConditions(server: McpServerCRD, deploymentReady: boolean): Promise<void>+ }+ ).updateStatusConditions(server, true)++ expect(customApi.getNamespacedCustomObjectStatus).toHaveBeenCalledTimes(1)+ expect(customApi.patchNamespacedCustomObjectStatus).not.toHaveBeenCalled()+ })})describe('unconfigured inventory authority fails closed (G1)', () => {- const appsApi = createMockAppsApi()- const coreApi = createMockCoreApi()- const customApi = createMockCustomApi()+ let appsApi: ReturnType<typeof createMockAppsApi>+ let coreApi: ReturnType<typeof createMockCoreApi>+ let customApi: ReturnType<typeof createMockCustomApi>let errorSpy: ReturnType<typeof vi.spyOn>// Deliberately NO assumeInventoryAuthorityWhenUnconfigured and NO@@ -2074,7 +2936,9 @@ describe('unconfigured inventory authority fails closed (G1)', () => {}beforeEach(() => {- vi.clearAllMocks()+ appsApi = createMockAppsApi()+ coreApi = createMockCoreApi()+ customApi = createMockCustomApi()errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})})diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex 64cd8ac76..5ce469df1 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -139,6 +139,50 @@ const DEFAULT_FULL_RECONCILE_MAX_CONCURRENCY = 10// Status is a read/merge/write transaction shared with other controllers.// Bound optimistic-conflict retries so a hot object cannot stall reconciliation.const STATUS_CONDITION_WRITE_MAX_ATTEMPTS = 3+/** Types HCC no longer writes and strips from every status write (#606). A future writer must not reuse `NetworkReady`; remove the entry first. */+const RETIRED_STATUS_CONDITION_TYPES = ['NetworkReady'] as const+export const RUNTIME_NOT_DESIRED_DISABLED_MESSAGE =+ 'McpServer is disabled; HCC does not run its runtime'+export const RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE =+ 'Env Secret validation failed; HCC does not run its runtime'++function isRetiredStatusConditionType(type: string): boolean {+ return (RETIRED_STATUS_CONDITION_TYPES as readonly string[]).includes(type)+}++type StatusWriteDecline = 'record-obligation' | 'silent'+type StatusWriteDeclineArg = () => StatusWriteDecline+type StatusWriteOutcome = 'wrote' | 'unchanged' | 'declined'+type RuntimeWithdrawKind = 'disabled' | 'fail-closed'+type DeploymentDeleteOutcome = 'gone' | 'denied' | 'foreign'++type RetractManagedRuntimeResult = {+ retracted: boolean+ cleanupComplete: boolean+ foreignRuntime: boolean+ runtimeGone: boolean+}++function shouldStopAfterIncompleteRetract(+ retract: RetractManagedRuntimeResult,+ isCurrent: () => boolean+): boolean {+ if (retract.retracted || retract.foreignRuntime) return false+ return !(retract.runtimeGone && isCurrent())+}++function runtimeNotDesiredMessage(kind: RuntimeWithdrawKind): string {+ switch (kind) {+ case 'disabled':+ return RUNTIME_NOT_DESIRED_DISABLED_MESSAGE+ case 'fail-closed':+ return RUNTIME_NOT_DESIRED_FAIL_CLOSED_MESSAGE+ default: {+ const _exhaustive: never = kind+ return _exhaustive+ }+ }+}function throwCleanupFailures(failures: unknown[], message: string): void {if (failures.length > 0) {@@ -257,6 +301,12 @@ interface ReadinessPollState {*/isCurrent: () => booleanserver: McpServerCRD+ /**+ * Snapshot of `readinessRetirements` at window create time (#606). A later+ * retirement increments the name's counter; the tick must not publish after+ * that, even if this state was already deleted from `readinessPolls`.+ */+ retirementEpoch: number}export class McpServerReconciler {@@ -302,6 +352,13 @@ export class McpServerReconciler {* sticky for the same Deployment generation.*/private readonly readinessPolls: Map<string, ReadinessPollState> = new Map()+ /**+ * Retirement epoch per server name (#606). Incremented when a disabled or+ * fail-closed reconcile retracts the readiness window. `clearStatus` does+ * not delete entries: a recreated name keeps the counter so stale ticks fail+ * the epoch check in addition to the uid fence.+ */+ private readonly readinessRetirements = new Map<string, number>()/*** Tail promise of the in-flight reconcile chain per server name. Serializes@@ -348,6 +405,26 @@ export class McpServerReconciler {}}+ /**+ * Retract the live readiness window for a name that will no longer run a+ * runtime (#606). Increments the retirement epoch first so an in-flight tick+ * that already deleted its map entry still fails `retiredNow()`. Clears any+ * pending timer. Does not record an incomplete-publication obligation. Poll+ * ticks pass an `onDecline` factory evaluated at decline time: silent only+ * when `retiredNow()` is true, otherwise the revision fence still records+ * an obligation.+ */+ private retireReadinessWindow(name: string): void {+ this.readinessRetirements.set(name, (this.readinessRetirements.get(name) ?? 0) + 1)+ const poll = this.readinessPolls.get(name)+ if (poll === undefined) return+ if (poll.timer !== undefined) {+ clearTimeout(poll.timer)+ poll.timer = undefined+ }+ this.readinessPolls.delete(name)+ }+constructor(kc: k8s.KubeConfig, deps?: McpServerReconcilerDeps) {this.appsApi = deps?.appsApi ?? kc.makeApiClient(k8s.AppsV1Api)this.coreApi = deps?.coreApi ?? kc.makeApiClient(k8s.CoreV1Api)@@ -618,6 +695,7 @@ export class McpServerReconciler {detail: previous?.detail ?? '',isCurrent,server,+ retirementEpoch: this.readinessRetirements.get(name) ?? 0,}this.readinessPolls.set(name, state)@@ -627,6 +705,11 @@ export class McpServerReconciler {// can tell "tick running" apart from "window ended" during our awaits.state.timer = undefinedstate.ticking = true+ const retiredNow = (): boolean =>+ (this.readinessRetirements.get(name) ?? 0) !== state.retirementEpoch+ const publishIfCurrent = (): boolean => state.isCurrent() && !retiredNow()+ const pollWriteDecline = (): StatusWriteDecline =>+ retiredNow() ? 'silent' : 'record-obligation'// Mutation fencing: a superseded reconcile (desired-revision or// inventory-authority change) must not keep observing — or writing// status — on behalf of a stale desired state. End the window quietly;@@ -692,10 +775,18 @@ export class McpServerReconciler {// always false, so this is the ONLY place a converged rollout is ever// recorded — without it every successful credential rotation would sit// at DeploymentReady=False forever and the UI would report a timeout.- await this.updateStatusConditions(state.server, true, undefined, state.isCurrent)+ if (retiredNow()) return+ await this.updateStatusConditions(+ state.server,+ true,+ undefined,+ publishIfCurrent,+ pollWriteDecline+ )// Readiness is decoupled from fleet convergence: the poll also// upgrades the persisted, generation-matched Ready condition so a// controller restart can trust the recorded readiness (see getStatus).+ if (retiredNow()) returnawait this.writeStatusCondition(state.server,{@@ -704,7 +795,8 @@ export class McpServerReconciler {reason: 'ReconcileSuccess',message: 'Deployment created',},- state.isCurrent+ publishIfCurrent,+ pollWriteDecline)return}@@ -725,7 +817,14 @@ export class McpServerReconciler {// Give up loudly, on the CRD: a stuck rollout must be visible to the// operator with the numbers that prove it, never left as a stale// condition or a silent Unknown.- await this.updateStatusConditions(state.server, false, rollout.detail, state.isCurrent)+ if (retiredNow()) return+ await this.updateStatusConditions(+ state.server,+ false,+ rollout.detail,+ publishIfCurrent,+ pollWriteDecline+ )return}@@ -1657,37 +1756,169 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvname: string,namespace: string,deleteAllowed?: () => Promise<boolean>- ): Promise<void> {+ ): Promise<DeploymentDeleteOutcome> {let deployment: k8s.V1Deploymenttry {deployment = await this.appsApi.readNamespacedDeployment({ name, namespace })} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[Reconciler] Deployment "${name}" already gone`)- } else {- console.error(`[Reconciler] Failed to read Deployment "${name}" ownership:`, error)- throw error+ hccLogger.info('Deployment already gone', { name })+ return 'gone'}- return+ hccLogger.error('Failed to read Deployment ownership', { name, err: error })+ throw error}if (!this.isHccOwnedMcpResource(deployment, name)) {- console.warn(`[Reconciler] Skipping Deployment "${name}" delete — not HCC-owned`)- return+ hccLogger.warn('Skipping Deployment delete — not HCC-owned', { name })+ return 'foreign'}- if (deleteAllowed && !(await deleteAllowed())) return+ if (deleteAllowed && !(await deleteAllowed())) return 'denied'try {await this.appsApi.deleteNamespacedDeployment({ name, namespace })- console.log(`[Reconciler] Deleted Deployment "${name}"`)+ hccLogger.info('Deleted Deployment', { name })+ return 'gone'} catch (error: unknown) {if (getErrorCode(error) === 404) {- console.log(`[Reconciler] Deployment "${name}" already gone`)- } else {- console.error(`[Reconciler] Failed to delete Deployment "${name}":`, error)- throw error+ hccLogger.info('Deployment already gone', { name })+ return 'gone'+ }+ hccLogger.error('Failed to delete Deployment', { name, err: error })+ throw error+ }+ }++ /**+ * Delete leftover HCC-owned ConfigMap and Service after the Deployment+ * outcome is known. Does not write status.+ */+ private async deleteHccOwnedRuntimeSiblings(+ name: string,+ namespace: string,+ deleteAllowed?: () => Promise<boolean>+ ): Promise<void> {+ const siblingFailures: unknown[] = []+ for (const cleanup of [+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),+ ]) {+ try {+ await cleanup()+ } catch (error) {+ siblingFailures.push(error)}}+ throwCleanupFailures(+ siblingFailures,+ `Failed to delete runtime Kubernetes resources for McpServer "${name}"`+ )+ }++ /**+ * Withdraw an HCC-owned runtime (#606): once the Deployment is confirmed+ * gone, retire the poll, persist DeploymentReady=False/RuntimeNotDesired,+ * then finish sibling deletes. Sibling failure does not roll back the+ * retract. Retire before the retract write so an in-flight tick cannot+ * republish True after the retract PATCH commits but before the epoch+ * bumps. A foreign (WRC-owned) Deployment retires the window without+ * writing RuntimeNotDesired.+ */+ private async retractManagedRuntime(+ server: McpServerCRD,+ kind: RuntimeWithdrawKind,+ isCurrent: () => boolean+ ): Promise<RetractManagedRuntimeResult> {+ const name = server.name+ const namespace = server.namespace+ const deleteAllowed = async (): Promise<boolean> => isCurrent()+ let cleanupComplete = true+ const gatedDeleteAllowed = async (): Promise<boolean> => {+ const allowed = await deleteAllowed()+ if (!allowed) cleanupComplete = false+ return allowed+ }++ let deploymentOutcome: DeploymentDeleteOutcome+ try {+ deploymentOutcome = await this.deleteDeploymentIfHccOwned(name, namespace, deleteAllowed)+ } catch (error) {+ const failures: unknown[] = [error]+ for (const cleanup of [+ () => this.deleteConfigMapIfHccOwned(`${name}-nginx-conf`, namespace, name, deleteAllowed),+ () => this.deleteServiceIfHccOwned(name, namespace, deleteAllowed),+ ]) {+ try {+ await cleanup()+ } catch (siblingError) {+ failures.push(siblingError)+ }+ }+ throwCleanupFailures(+ failures,+ `Failed to delete runtime Kubernetes resources for McpServer "${name}"`+ )+ throw new Error(`Failed to delete runtime Kubernetes resources for McpServer "${name}"`)+ }++ switch (deploymentOutcome) {+ case 'denied':+ return {+ retracted: false,+ cleanupComplete: false,+ foreignRuntime: false,+ runtimeGone: false,+ }+ case 'foreign':+ this.retireReadinessWindow(name)+ await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)+ return {+ retracted: false,+ cleanupComplete: cleanupComplete && isCurrent(),+ foreignRuntime: true,+ runtimeGone: false,+ }+ case 'gone':+ break+ default: {+ const _exhaustive: never = deploymentOutcome+ return _exhaustive+ }+ }++ this.retireReadinessWindow(name)+ const retractWrite = await this.writeStatusCondition(+ server,+ {+ type: 'DeploymentReady',+ status: 'False',+ reason: 'RuntimeNotDesired',+ message: runtimeNotDesiredMessage(kind),+ },+ isCurrent+ )+ const epoch = this.readinessRetirements.get(name)+ if (retractWrite === 'declined') {+ hccLogger.warn('Retract write declined', { name, kind, epoch, outcome: retractWrite })+ } else {+ hccLogger.info('Retracted managed runtime', { name, kind, epoch, outcome: retractWrite })+ }++ await this.deleteHccOwnedRuntimeSiblings(name, namespace, gatedDeleteAllowed)+ if (retractWrite === 'declined') {+ return {+ retracted: false,+ cleanupComplete: false,+ foreignRuntime: false,+ runtimeGone: true,+ }+ }+ return {+ retracted: true,+ cleanupComplete: cleanupComplete && isCurrent(),+ foreignRuntime: false,+ runtimeGone: true,+ }}private async deleteConfigMapIfHccOwned(@@ -1831,9 +2062,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv}/**- * G11: Update status conditions on McpServer CRD.- * Sets NetworkReady and DeploymentReady conditions so WRC can watch- * for status changes instead of polling with sleeps.+ * G11: Update DeploymentReady on the McpServer status subresource.+ * The condition describes Deployment rollout. WRC does not watch this type;+ * it reads the `clerum.io/network-ready` annotation.*/private async updateStatusConditions(server: McpServerCRD,@@ -1844,19 +2075,10 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* that only know the verdict omit it.*/rolloutDetail?: string,- isCurrent: () => boolean = () => true+ isCurrent: () => boolean = () => true,+ onDecline: StatusWriteDeclineArg = () => 'record-obligation'): Promise<void> {try {- const networkWrote = await this.writeStatusCondition(- server,- {- type: 'NetworkReady',- status: 'True',- reason: 'NetworkPoliciesApplied',- message: 'NetworkPolicies and Service created',- },- isCurrent- )const deploymentWrote = await this.writeStatusCondition(server,{@@ -1873,16 +2095,18 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv? 'Deployment has ready replicas': 'Waiting for pods to become ready',},- isCurrent+ isCurrent,+ onDecline)- if (networkWrote || deploymentWrote) {- console.log(- `[Reconciler] Updated status conditions on "${server.name}" (DeploymentReady=${deploymentReady})`- )+ if (deploymentWrote === 'wrote') {+ hccLogger.info('Updated status conditions', {+ name: server.name,+ deploymentReady,+ })}} catch (error) {// Status subresource may not exist yet — log but don't fail reconciliation- console.warn(`[Reconciler] Failed to update status conditions on "${server.name}":`, error)+ hccLogger.warn('Failed to update status conditions', { name: server.name, err: error })}}@@ -1892,24 +2116,41 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv* Merge semantics:* - Condition with same `type` and unchanged `status` → keep existing* `lastTransitionTime`, refresh `reason` / `message`.+ * - `DeploymentReady` with the same `status` but a new `reason` or+ * `message` → bump `lastTransitionTime` so recovery after+ * `RuntimeNotDesired` is visible to #223 consumers.* - Condition with same `type` but different `status` → bump* `lastTransitionTime` to now.* - New type → append with fresh `lastTransitionTime`.+ * - Any write also strips retired types (NetworkReady). If a retired+ * type is still present, the patch is forced even when the requested+ * condition is otherwise unchanged.** Side-effect: updates the `clerum_hcc_mcpserver_missing_secret` gauge for* `SecretResolved` conditions (1 when False, 0 when True).*+ * `onDecline` is a factory evaluated at decline time. The default records+ * an incomplete-publication obligation. Poll ticks pass a factory that is+ * `'silent'` only while `retiredNow()` is true.+ *+ * Returns `'wrote'` after a successful patch, `'unchanged'` when the+ * requested condition already matches and no retired type remains, or+ * `'declined'` when the fence or a write error stops the update.+ ** Failure modes:- * - 404 (CRD deleted mid-reconcile) → log + swallow.+ * - 404 (CRD deleted mid-reconcile) → log + `decline()`.* - 409/422 (optimistic conflict) → re-read, re-merge, bounded retry.- * - Other errors → log + swallow (status writes are best-effort).+ * - Other errors or exhausted retries → log + `decline()`.*/async writeStatusCondition(server: McpServerCRD,condition: Omit<McpServerCondition, 'lastTransitionTime'>,- isCurrent: () => boolean = () => true- ): Promise<boolean> {- if (!isCurrent()) return this.recordIncompletePublication(server)+ isCurrent: () => boolean = () => true,+ onDecline: StatusWriteDeclineArg = () => 'record-obligation'+ ): Promise<StatusWriteOutcome> {+ const decline = (): StatusWriteOutcome =>+ onDecline() === 'silent' ? 'declined' : this.recordIncompletePublication(server)+ if (!isCurrent()) return decline()const now = new Date().toISOString()// Update the missing-secret gauge for SecretResolved conditions.@@ -1934,7 +2175,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// if anything changed since our read; we then re-read and retry, so no// writer's condition is lost.for (let attempt = 1; attempt <= STATUS_CONDITION_WRITE_MAX_ATTEMPTS; attempt += 1) {- if (!isCurrent()) return this.recordIncompletePublication(server)+ if (!isCurrent()) return decline()// Re-read on every optimistic-conflict retry. Merging against the// latest resourceVersion preserves conditions written by peer// controllers between our read and patch.@@ -1960,15 +2201,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — skipping status update`)- return this.recordIncompletePublication(server)+ return decline()}console.warn(`[Reconciler] Failed to read status for "${server.name}" — skipping status update for type "${condition.type}" to avoid clobbering other status fields:`,error)- return this.recordIncompletePublication(server)+ return decline()}- if (!isCurrent()) return this.recordIncompletePublication(server)+ if (!isCurrent()) return decline()const prior = existingConditions.find(c => c.type === condition.type)const observedGeneration = condition.observedGeneration ?? server.generation@@ -1977,7 +2218,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvprior.status === condition.status &&prior.reason === condition.reason &&prior.message === condition.message &&- prior.observedGeneration === observedGeneration+ prior.observedGeneration === observedGeneration &&+ !existingConditions.some(existing => isRetiredStatusConditionType(existing.type))// Skip the patch when the condition is identical to what's already on// the CRD. Without this, every reconcile bumps the McpServer's@@ -1985,11 +2227,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// reconcile — a tight self-loop that also amplifies NetworkPolicy// optimistic-lock contention downstream.if (unchanged) {- return false+ return 'unchanged'}+ const refreshDeploymentReadyTime =+ condition.type === 'DeploymentReady' &&+ prior !== undefined &&+ (prior.reason !== condition.reason || prior.message !== condition.message)const lastTransitionTime =- prior && prior.status === condition.status ? prior.lastTransitionTime : now+ prior && prior.status === condition.status && !refreshDeploymentReadyTime+ ? prior.lastTransitionTime+ : nowconst merged: McpServerCondition = {type: condition.type,@@ -2000,7 +2248,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv...(observedGeneration !== undefined && { observedGeneration }),}- const nextConditions = [...existingConditions.filter(c => c.type !== condition.type), merged]+ const nextConditions = [+ ...existingConditions.filter(+ existing =>+ existing.type !== condition.type && !isRetiredStatusConditionType(existing.type)+ ),+ merged,+ ]const writePreconditions: Array<{op: 'test'@@ -2039,7 +2293,7 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// The generated client prefers JSON Patch for this endpoint. Send an// actual patch document so status writes keep working across client// upgrades instead of relying on merge-patch object bodies.- if (!isCurrent()) return this.recordIncompletePublication(server)+ if (!isCurrent()) return decline()await this.customApi.patchNamespacedCustomObjectStatus({group: 'clerum.io',version: 'v1alpha1',@@ -2048,14 +2302,14 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvname: server.name,body: statusPatch,})- return true+ return 'wrote'} catch (error) {const errorCode = getErrorCode(error)if (errorCode === 404) {console.warn(`[Reconciler] McpServer "${server.name}" deleted mid-reconcile — status patch skipped`)- return this.recordIncompletePublication(server)+ return decline()}// 409 (Conflict) or 422 (a `test` op failed) mean a concurrent writer// won the race. Re-read and retry rather than lose our condition. Any@@ -2071,18 +2325,17 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv`[Reconciler] Failed to write status condition ${condition.type}=${condition.status} on "${server.name}" (attempt ${attempt}/${STATUS_CONDITION_WRITE_MAX_ATTEMPTS}):`,error)- return this.recordIncompletePublication(server)+ return decline()}}- return this.recordIncompletePublication(server)+ return decline()}- private recordIncompletePublication(server: McpServerCRD): false {- // Keep the changed-boolean API: an unchanged condition is successful but- // returns false. Only a failed/retired write replaces the obligation, so- // an enclosing reconcile cannot clear that failure on normal return.+ private recordIncompletePublication(server: McpServerCRD): 'declined' {+ // A declined or failed write replaces the obligation so an enclosing+ // reconcile cannot clear that failure on a later successful sibling write.this.pendingReconciliations.set(server.name, { namespace: server.namespace })- return false+ return 'declined'}// ─── Public API ─────────────────────────────────────────────────────@@ -2155,16 +2408,15 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// If disabled, respect ownership before any cleanup. managed:false means// WRC owns runtime resources; HCC only marks discovery status disabled.if (server.spec.enabled === false) {+ let cleanupComplete = true+ let retract: RetractManagedRuntimeResult | undefinedif (currentManaged) {console.log(`[Reconciler] McpServer "${server.name}" is disabled — removing HCC-owned resources`)- if (- !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>- isCurrent()- ))- )- return false+ retract = await this.retractManagedRuntime(server, 'disabled', isCurrent)+ if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false+ cleanupComplete = retract.cleanupComplete} else {console.log(`[Reconciler] McpServer "${server.name}" is disabled but WRC-owned; skipping runtime cleanup`@@ -2182,7 +2434,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- return isCurrent()+ if (retract && !retract.retracted && !retract.foreignRuntime) return false+ return cleanupComplete && isCurrent()}if (!currentManaged) {@@ -2196,13 +2449,13 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvconsole.error(`[Reconciler] Skipping deployment of "${server.name}" — secret validation failed`)- if (shouldFailClosedForSecretFailure(secretResult.reason)) {- if (- !(await this.deleteRuntimeResources(server.name, server.namespace, async () =>- isCurrent()- ))- )- return false+ const withdrawRuntime = shouldFailClosedForSecretFailure(secretResult.reason)+ let cleanupComplete = true+ let retract: RetractManagedRuntimeResult | undefined+ if (withdrawRuntime) {+ retract = await this.retractManagedRuntime(server, 'fail-closed', isCurrent)+ if (shouldStopAfterIncompleteRetract(retract, isCurrent)) return false+ cleanupComplete = retract.cleanupComplete} else {console.warn(`[Reconciler] Preserving existing runtime for "${server.name}" after transient Secret read failure`@@ -2234,7 +2487,8 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv},isCurrent)- return shouldFailClosedForSecretFailure(secretResult.reason) && isCurrent()+ if (retract && !retract.retracted && !retract.foreignRuntime) return false+ return withdrawRuntime && cleanupComplete && isCurrent()}// Ensure resources exist and are up to date@@ -2344,9 +2598,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvawait this.updateStatusConditions(server, ready, exhaustedDetail, isCurrent)// PR-B B1: record SecretResolved=True + Ready=True after a successful- // reconcile. This is additive to updateStatusConditions (NetworkReady /- // DeploymentReady); the merge logic in writeStatusCondition preserves- // the existing conditions by type.+ // reconcile. This is additive to updateStatusConditions (DeploymentReady);+ // the merge logic in writeStatusCondition preserves the existing+ // conditions by type.if (server.spec.envSecret) {await this.writeStatusCondition(server,diff --git a/scripts/e2e/run-vitest-e2e.sh b/scripts/e2e/run-vitest-e2e.shindex 295a92ae0..c4ebbeb70 100755--- a/scripts/e2e/run-vitest-e2e.sh+++ b/scripts/e2e/run-vitest-e2e.sh@@ -34,6 +34,7 @@ DEFAULT_CLUSTER_VITEST_SUITES=(integration/mcp-secret-rotation-isolation.test.tsintegration/mcp-secret-rotation-shared.test.tsintegration/mcp-secret-rotation-remote.test.ts+ integration/mcp-runtime-conditions.test.tsintegration/mcp-hcc-upgrade-rollout.test.tsintegration/mcp-server-readiness-regression.test.tsintegration/mcp-secrets-access-guard.test.tsdiff --git a/tests/e2e/integration/mcp-runtime-conditions.test.ts b/tests/e2e/integration/mcp-runtime-conditions.test.tsnew file mode 100644index 000000000..d5dd8a73f--- /dev/null+++ b/tests/e2e/integration/mcp-runtime-conditions.test.ts@@ -0,0 +1,190 @@+/**+ * E2E: HCC runtime conditions (#606).+ *+ * Proves, on a live cluster, that HCC retracts DeploymentReady when it does+ * not run a managed server and never writes a NetworkReady condition.+ *+ * Actions go through control-api (DELETE + POST of the Secret; full-spec PUT+ * to disable). Do not write the Secret with kubectl after setup, and do not+ * merge-patch it — PUT is merge-patch and cannot produce a missing key.+ *+ * Run:+ * bash scripts/e2e/run-vitest-e2e.sh integration/mcp-runtime-conditions.test.ts+ */+import { afterAll, beforeAll, describe, expect, it } from 'vitest'+import {+ MCP_SERVERS_NAMESPACE,+ ROLLOUT_TIMEOUT_MS,+ adminLogin,+ conditionTransitionMs,+ deleteMcpSecret,+ deleteMcpServerFixture,+ getMcpServerResource,+ kubectlApply,+ localMcpServerYaml,+ postMcpSecret,+ putMcpServer,+ randomSuffix,+ requireControlApiUp,+ requireDeploymentAbsent,+ requireHccDevModeUnset,+ secretYaml,+ waitForRolloutCondition,+ waitForStatusCondition,+} from './mcpCredentialRotation.helpers.js'++const RUN_ID = randomSuffix()+const SERVER_NAME = `e2e-runtime-cond-${RUN_ID}`+const SECRET_NAME = `${SERVER_NAME}-credentials`+const SECRET_KEY = 'api-key'+const INITIAL_SECRET_VALUE = 'initial-valid-value'+const RESTORED_SECRET_VALUE = 'restored-valid-value'++let controlApiUp = false+let adminToken = ''++function expectNoNetworkReady(resource: {+ status?: { conditions?: Array<{ type: string }> }+}): void {+ const conditions = resource.status?.conditions+ expect(conditions, 'status.conditions must be present').toBeDefined()+ const types = (conditions ?? []).map(condition => condition.type)+ expect(types, `DeploymentReady must be present; saw ${types.join(',')}`).toContain(+ 'DeploymentReady'+ )+ expect(types, `NetworkReady must be absent; saw ${types.join(',')}`).not.toContain('NetworkReady')+}++beforeAll(async () => {+ controlApiUp = await requireControlApiUp('mcp-runtime-conditions')+ if (!controlApiUp) return++ requireHccDevModeUnset()+ adminToken = await adminLogin()++ kubectlApply(+ secretYaml(SECRET_NAME, MCP_SERVERS_NAMESPACE, {+ [SECRET_KEY]: INITIAL_SECRET_VALUE,+ })+ )+ kubectlApply(+ localMcpServerYaml({+ name: SERVER_NAME,+ envSecretName: SECRET_NAME,+ keys: [{ secretKey: SECRET_KEY, envVar: 'E2E_ROTATION_API_KEY' }],+ })+ )+}, ROLLOUT_TIMEOUT_MS + 30_000)++afterAll(() => {+ if (!controlApiUp) return+ deleteMcpServerFixture(SERVER_NAME, SECRET_NAME)+})++describe('mcp runtime conditions (#606)', () => {+ it(+ 'retracts DeploymentReady on Secret delete, restores it on recreate, and disables the runtime',+ async () => {+ if (!controlApiUp) return++ const ready = await waitForRolloutCondition(SERVER_NAME, adminToken, {+ expectStatus: 'True',+ expectReason: 'ReplicasAvailable',+ sinceMs: 0,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))++ const deletedAt = conditionTransitionMs(ready)+ const deleted = await deleteMcpSecret(SECRET_NAME, adminToken)+ expect(deleted.status, `DELETE secret failed: ${JSON.stringify(deleted.data)}`).toBe(200)++ await waitForStatusCondition(SERVER_NAME, adminToken, {+ type: 'SecretResolved',+ expectStatus: 'False',+ expectReason: 'SecretNotFound',+ sinceMs: deletedAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ await waitForStatusCondition(SERVER_NAME, adminToken, {+ type: 'Ready',+ expectStatus: 'False',+ expectReason: 'SecretValidationFailed',+ sinceMs: deletedAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ const retired = await waitForRolloutCondition(SERVER_NAME, adminToken, {+ expectStatus: 'False',+ expectReason: 'RuntimeNotDesired',+ sinceMs: deletedAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ expect(retired.message).toContain('Env Secret validation failed')+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))+ requireDeploymentAbsent(SERVER_NAME, MCP_SERVERS_NAMESPACE)++ const restoredAt = conditionTransitionMs(retired)+ const created = await postMcpSecret(+ SECRET_NAME,+ { [SECRET_KEY]: RESTORED_SECRET_VALUE },+ adminToken+ )+ expect(created.status, `POST secret failed: ${JSON.stringify(created.data)}`).toBe(201)++ const restored = await waitForRolloutCondition(SERVER_NAME, adminToken, {+ expectStatus: 'True',+ expectReason: 'ReplicasAvailable',+ sinceMs: restoredAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ expect(Date.parse(restored.lastTransitionTime)).toBeGreaterThan(+ Date.parse(retired.lastTransitionTime)+ )+ const recoveredReady = await waitForStatusCondition(SERVER_NAME, adminToken, {+ type: 'Ready',+ expectStatus: 'True',+ expectReason: 'ReconcileSuccess',+ sinceMs: restoredAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ await waitForStatusCondition(SERVER_NAME, adminToken, {+ type: 'SecretResolved',+ expectStatus: 'True',+ expectReason: 'SecretFound',+ sinceMs: restoredAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))++ const current = await getMcpServerResource(SERVER_NAME, adminToken)+ const disabledAt = conditionTransitionMs(recoveredReady)+ const disabled = await putMcpServer(+ SERVER_NAME,+ {+ metadata: { resourceVersion: current.metadata.resourceVersion },+ spec: { ...current.spec, enabled: false },+ },+ adminToken+ )+ expect(disabled.status, `PUT disable failed: ${JSON.stringify(disabled.data)}`).toBe(200)++ await waitForStatusCondition(SERVER_NAME, adminToken, {+ type: 'Ready',+ expectStatus: 'False',+ expectReason: 'Disabled',+ sinceMs: disabledAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ const disabledReady = await waitForRolloutCondition(SERVER_NAME, adminToken, {+ expectStatus: 'False',+ expectReason: 'RuntimeNotDesired',+ sinceMs: disabledAt,+ timeoutMs: ROLLOUT_TIMEOUT_MS,+ })+ expect(disabledReady.message).toContain('McpServer is disabled')+ expectNoNetworkReady(await getMcpServerResource(SERVER_NAME, adminToken))+ requireDeploymentAbsent(SERVER_NAME, MCP_SERVERS_NAMESPACE)+ },+ ROLLOUT_TIMEOUT_MS * 8 + 30_000+ )+})diff --git a/tests/e2e/integration/mcpCredentialRotation.helpers.ts b/tests/e2e/integration/mcpCredentialRotation.helpers.tsindex e2bfa9b71..6627640d3 100644--- a/tests/e2e/integration/mcpCredentialRotation.helpers.ts+++ b/tests/e2e/integration/mcpCredentialRotation.helpers.ts@@ -323,6 +323,70 @@ export async function putMcpSecret()}+export async function deleteMcpSecret(+ name: string,+ session: string+): Promise<{ status: number; data: unknown }> {+ return deleteJson(+ `${CONTROL_API_URL}/api/v1/admin/mcp-secrets/${encodeURIComponent(name)}`,+ adminSessionHeader(session)+ )+}++export async function postMcpSecret(+ name: string,+ data: Record<string, string>,+ session: string+): Promise<{ status: number; data: unknown }> {+ return postJson(+ `${CONTROL_API_URL}/api/v1/admin/mcp-secrets`,+ { name, data },+ adminSessionHeader(session)+ )+}++export async function putMcpServer(+ name: string,+ body: { metadata?: { resourceVersion?: string }; spec: Record<string, unknown> },+ session: string+): Promise<{ status: number; data: unknown }> {+ return putJson(+ `${CONTROL_API_URL}/api/v1/admin/mcp-servers/${encodeURIComponent(name)}`,+ body,+ adminSessionHeader(session)+ )+}++/** Fail loud when HCC is running with the Secret informer disabled. */+export function requireHccDevModeUnset(): void {+ const raw = kubectl(+ `-n control-plane get deploy host-context-controller -o jsonpath='{.spec.template.spec.containers[?(@.name=="host-context-controller")].env[?(@.name=="CLERUM_DEV_MODE")].value}'`+ )+ // Match host-context-controller/src/config.ts getEnvBool: true/TRUE/1.+ if (raw.toLowerCase() === 'true' || raw === '1') {+ throw new Error(+ 'HCC pod has CLERUM_DEV_MODE enabled; the Secret informer is off. Stop and fix the profile before running runtime-condition e2e.'+ )+ }+}++/** Milliseconds of a persisted condition timestamp. Throws if unparseable. */+export function conditionTransitionMs(condition: { lastTransitionTime: string }): number {+ const parsed = Date.parse(condition.lastTransitionTime)+ if (Number.isNaN(parsed)) {+ throw new Error(`invalid lastTransitionTime: ${condition.lastTransitionTime}`)+ }+ return parsed+}++/** Empty name means the Deployment is gone. Any kubectl error throws. */+export function requireDeploymentAbsent(name: string, namespace: string): void {+ const found = kubectl(`get deploy ${name} -n ${namespace} --ignore-not-found -o name`)+ if (found !== '') {+ throw new Error(`expected Deployment "${name}" to be absent, found ${found}`)+ }+}+export async function getMcpServerResource(name: string,session: string@@ -346,29 +410,39 @@ export function deploymentReadyCondition(}/**- * Poll GET /admin/mcp-servers/:name until DeploymentReady reaches- * `expectStatus` with a `lastTransitionTime` strictly after `sinceMs`.+ * Poll GET /admin/mcp-servers/:name until a status condition reaches+ * `expectStatus` (and optional `expectReason`) with a `lastTransitionTime`+ * at or after `sinceMs`.*- * The `sinceMs` correlation is load-bearing (plan Fase 3, requisito 6 /- * hallazgo 1): without it, a condition left over from a PREVIOUS deploy would- * satisfy the predicate instantly, before the rollout under test even began.+ * The `sinceMs` correlation is load-bearing: without it, a leftover+ * condition from a previous deploy would satisfy the predicate instantly,+ * before the rollout under test even began. Prefer a previous condition's+ * server-side timestamp over the host clock so VM clock skew cannot flake.*/-export async function waitForRolloutCondition(+export async function waitForStatusCondition(name: string,token: string,- opts: { expectStatus: 'True' | 'False'; sinceMs: number; timeoutMs: number }+ opts: {+ type: string+ expectStatus: 'True' | 'False'+ expectReason?: string+ sinceMs: number+ timeoutMs: number+ }): Promise<McpServerCondition> {+ const reasonLabel = opts.expectReason ? ` reason=${opts.expectReason}` : ''return waitFor(- `McpServer "${name}" DeploymentReady=${opts.expectStatus} with lastTransitionTime after ${new Date(+ `McpServer "${name}" ${opts.type}=${opts.expectStatus}${reasonLabel} with lastTransitionTime at or after ${new Date(opts.sinceMs).toISOString()}`,async () => {const resource = await getMcpServerResource(name, token)- const cond = deploymentReadyCondition(resource)+ const cond = resource.status?.conditions?.find(c => c.type === opts.type)if (!cond) return nullconst transitionMs = Date.parse(cond.lastTransitionTime)if (Number.isNaN(transitionMs) || transitionMs < opts.sinceMs) return nullif (cond.status !== opts.expectStatus) return null+ if (opts.expectReason !== undefined && cond.reason !== opts.expectReason) return nullreturn cond},opts.timeoutMs,@@ -382,6 +456,27 @@ export async function waitForRolloutCondition()}+/**+ * Poll GET /admin/mcp-servers/:name until DeploymentReady reaches+ * `expectStatus` with a `lastTransitionTime` at or after `sinceMs`.+ *+ * The `sinceMs` correlation is load-bearing: without it, a leftover+ * condition from a previous deploy would satisfy the predicate instantly,+ * before the rollout under test even began.+ */+export async function waitForRolloutCondition(+ name: string,+ token: string,+ opts: {+ expectStatus: 'True' | 'False'+ expectReason?: string+ sinceMs: number+ timeoutMs: number+ }+): Promise<McpServerCondition> {+ return waitForStatusCondition(name, token, { type: 'DeploymentReady', ...opts })+}+/*** E2's guardian: poll until the rollout reaches a TERMINAL failure* (`DeploymentReady=False`, `reason=RolloutIncomplete`) after `sinceMs`. If adiff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.jsonindex a50f0aec9..7d7aecf0b 100644--- a/tests/e2e/package-lock.json+++ b/tests/e2e/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-e2e-tests",- "version": "0.1.80",+ "version": "0.1.83","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-e2e-tests",- "version": "0.1.80",+ "version": "0.1.83","devDependencies": {"@types/jsonwebtoken": "^9.0.10","@types/node": "^22.0.0",diff --git a/tests/e2e/package.json b/tests/e2e/package.jsonindex 5d00434de..aad2a7b47 100644--- a/tests/e2e/package.json+++ b/tests/e2e/package.json@@ -1,6 +1,6 @@{"name": "clerum-e2e-tests",- "version": "0.1.80",+ "version": "0.1.83","private": true,"type": "module","scripts": {