Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
feat(hcc): converge existing host and filesystem workloads before create
Alfredo Lopezb085237905a3
diff --git a/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts b/control-ui/e2e/qa-recorder-shared-fs-folders.spec.tsindex 3862221e4..7ae14ca05 100644--- a/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts+++ b/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts@@ -1,8 +1,4 @@-// control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts-//-// Optional QA recorder journey (MUTATING). Requires QA_RECORDER_CONFIRM_MUTATIONS=1.-// Provisions a SharedFileSystem, then creates and renames a folder in its file-// browser. The SharedFileSystem is removed via Control API in finally.+// UI owns setup. API calls are limited to cleanup and negative authorization.import { expect, test } from '@playwright/test'import {CONTROL_API_URL,@@ -16,99 +12,221 @@ import {uniqueE2EName,} from './qa-recorder-helpers'-function sharedFsDeletePath(name: string): string {- return `/api/v1/admin/shared-filesystems/${encodeURIComponent(name)}`+const collectionPath = '/api/v1/admin/shared-filesystems'+type Filesystem = {+ metadata?: { name?: string; uid?: string }+ spec?: { size?: string; accessModes?: string[]; retainOnDelete?: boolean }+}+function isApiPath(rawUrl: string, path: string): boolean {+ const url = new URL(rawUrl)+ return url.origin === new URL(CONTROL_UI_URL).origin && url.pathname === `/control-api${path}`}test.describe('optional QA recorder: Control UI SharedFileSystem folders', () => {- test('SharedFileSystem — folder create + rename', async ({ page }, testInfo) => {+ test('SharedFileSystem — visible creation, folder rename and persistence', async ({+ page,+ }, testInfo) => {requireRecorderConfirm('QA_RECORDER_CONFIRM_MUTATIONS',- 'This journey provisions a SharedFileSystem and creates/renames a folder in it.'+ 'This journey creates a SharedFileSystem and renames a folder.')assertAllowedTarget('CONTROL_UI_URL', CONTROL_UI_URL)assertAllowedTarget('CONTROL_API_URL', CONTROL_API_URL)-- const credentials = adminCredentials()- const sharedFsName = uniqueE2EName('qa-recorder-shared-fs')- const folderName = 'qa-recorder-folder'- const renamedFolderName = 'qa-recorder-folder-renamed'-+ const name = uniqueE2EName('qa-recorder-shared-fs')+ const detailPath = `/agent-files/${encodeURIComponent(name)}`+ const apiPath = `${collectionPath}/${encodeURIComponent(name)}`+ const filesPath = `${apiPath}/proxy/v1/files`+ const folder = 'qa-recorder-folder'+ const renamed = 'qa-recorder-folder-renamed'+ let createdUid: string | undefinedtry {- await loginThroughUi(page, credentials)-- await page.goto(`${CONTROL_UI_URL}/agent-files/new`)- await expect(- page.getByRole('heading', { name: 'Create SharedFileSystem', exact: true })- ).toBeVisible({ timeout: 20_000 })-- await page.getByPlaceholder('team-mission').fill(sharedFsName)- await page.getByLabel('Shared filesystem size').fill('5')- await page.getByLabel('Shared filesystem size unit').selectOption({ value: 'Gi' })- await page- .locator('.cu-create-actions')- .getByRole('button', { name: 'Continue', exact: true })- .click()-- await expect(page.getByText('Access and retention', { exact: true })).toBeVisible({- timeout: 20_000,+ await test.step('sign in and navigate to Agent Files', async () => {+ await loginThroughUi(page, adminCredentials())+ const nav = page.getByRole('navigation', { name: 'Main sections' })+ await expect(nav).toBeVisible()+ await nav.getByRole('link', { name: 'Files', exact: true }).click()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)+ await expect(page.getByRole('heading', { name: /^Agent Files/ })).toBeVisible()+ await expect(page.getByRole('button', { name: 'New', exact: true })).toBeEnabled()})- await page.getByLabel('Access mode').selectOption({ value: 'ReadWriteOnce' })- await page.getByLabel(/Keep storage after deletion/i).check()- await page- .locator('.cu-create-actions')- .getByRole('button', { name: 'Continue', exact: true })- .click()-- await expect(page.getByText('Initial folders', { exact: true })).toBeVisible({- timeout: 20_000,+ await test.step('complete each wizard step and verify retained values', async () => {+ await page.getByRole('button', { name: 'New', exact: true }).click()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files/new`)+ await expect(+ page.getByRole('heading', {+ name: 'Create SharedFileSystem',+ exact: true,+ })+ ).toBeVisible()+ await expect(page.getByRole('region', { name: 'Storage request' })).toBeVisible()+ await page.getByLabel('Name', { exact: true }).fill(name)+ await page.getByLabel('Shared filesystem size', { exact: true }).fill('5')+ await page.getByLabel('Shared filesystem size unit').selectOption('Gi')+ await page.getByRole('button', { name: 'Continue', exact: true }).click()+ await expect(page.getByRole('region', { name: 'Access and retention' })).toBeVisible()+ await page.getByLabel('Access mode').selectOption('ReadWriteOnce')+ await page.getByLabel('Keep storage after deletion').uncheck()+ await expect(page.getByLabel('Access mode')).toHaveValue('ReadWriteOnce')+ await expect(page.getByLabel('Keep storage after deletion')).not.toBeChecked()+ await page.getByRole('button', { name: 'Back', exact: true }).click()+ await expect(page.getByRole('region', { name: 'Storage request' })).toBeVisible()+ await expect(page.getByLabel('Name', { exact: true })).toHaveValue(name)+ await expect(page.getByLabel('Shared filesystem size', { exact: true })).toHaveValue('5')+ await expect(page.getByLabel('Shared filesystem size unit')).toHaveValue('Gi')+ await page.getByRole('button', { name: 'Continue', exact: true }).click()+ await expect(page.getByRole('region', { name: 'Access and retention' })).toBeVisible()+ await expect(page.getByLabel('Keep storage after deletion')).not.toBeChecked()+ await page.getByRole('button', { name: 'Continue', exact: true }).click()+ await expect(page.getByRole('region', { name: 'Initial folders' })).toBeVisible()+ await expect(page.getByLabel('Directory name')).toBeVisible()+ await expect(page.getByText('No folders added.', { exact: true })).toBeVisible()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files/new`)})- await page- .locator('.cu-create-actions')- .getByRole('button', { name: 'Create', exact: true })- .click()-- await expect(- page.getByText(`Shared filesystem "${sharedFsName}" created.`, { exact: true })- ).toBeVisible({ timeout: 20_000 })-- await page.goto(`${CONTROL_UI_URL}/agent-files/${encodeURIComponent(sharedFsName)}`)- await expect(page.getByRole('heading', { name: sharedFsName, exact: true })).toBeVisible({- timeout: 20_000,+ await test.step('submit and observe the real transition to the list', async () => {+ const creation = page.waitForResponse(+ r =>+ isApiPath(r.url(), collectionPath) &&+ r.request().method() === 'POST' &&+ r.request().postDataJSON()?.name === name+ )+ await page.getByRole('button', { name: 'Create', exact: true }).click()+ const response = await creation+ expect(response.status()).toBe(201)+ const resource = (await response.json()) as Filesystem+ expect(resource.metadata?.name).toBe(name)+ expect(resource.metadata?.uid).toEqual(expect.any(String))+ createdUid = resource.metadata!.uid+ expect(createdUid).not.toBe('')+ expect(resource.spec).toMatchObject({+ size: '5Gi',+ accessModes: ['ReadWriteOnce'],+ retainOnDelete: false,+ })+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)+ await expect(page.getByRole('heading', { name: /^Agent Files/ })).toBeVisible()+ const row = page.getByRole('link', {+ name: `Open shared filesystem ${name}`,+ exact: true,+ })+ await expect(row).toBeVisible()+ await expect(row).toContainText('5Gi')})- await expect(page.getByRole('button', { name: 'New folder', exact: true })).toBeVisible({- timeout: 20_000,+ await test.step('refresh until Ready and open the recorded row', async () => {+ const row = page.getByRole('link', {+ name: `Open shared filesystem ${name}`,+ exact: true,+ })+ // The list has no auto-poll: exercise its visible Refresh action.+ await expect(async () => {+ await page.getByRole('button', { name: 'Refresh agent files', exact: true }).click()+ await expect(row.getByText('Ready', { exact: true })).toBeVisible({+ timeout: 1000,+ })+ }).toPass({ timeout: 180_000, intervals: [1000, 2000, 5000] })+ const listing = page.waitForResponse(+ r => isApiPath(r.url(), filesPath) && r.request().method() === 'GET'+ )+ await row.click()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}${detailPath}`)+ await expect(page.getByRole('heading', { name, exact: true })).toBeVisible()+ expect((await listing).status()).toBe(200)+ await expect(page.getByRole('button', { name: 'New folder', exact: true })).toBeEnabled()})-- await page.getByRole('button', { name: 'New folder', exact: true }).click()- const newFolderDialog = page.getByRole('dialog', { name: 'New folder', exact: true })- await expect(newFolderDialog).toBeVisible({ timeout: 20_000 })- await newFolderDialog.getByLabel('Folder name').fill(folderName)- await newFolderDialog.getByRole('button', { name: 'Create', exact: true }).click()-- await expect(page.getByRole('button', { name: folderName, exact: true })).toBeVisible({- timeout: 20_000,+ await test.step('create and rename a folder with visible controls', async () => {+ await page.getByRole('button', { name: 'New folder', exact: true }).click()+ const dialog = page.getByRole('dialog', {+ name: 'New folder',+ exact: true,+ })+ await expect(dialog).toBeVisible()+ await dialog.getByLabel('Folder name').fill(folder)+ const mkdir = page.waitForResponse(+ r => isApiPath(r.url(), `${filesPath}/mkdir`) && r.request().method() === 'POST'+ )+ await dialog.getByRole('button', { name: 'Create', exact: true }).click()+ expect((await mkdir).ok()).toBe(true)+ await expect(dialog).toBeHidden()+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeVisible()+ await page.getByRole('button', { name: `Actions for ${folder}`, exact: true }).click()+ await page.getByRole('menuitem', { name: 'Rename', exact: true }).click()+ const rename = page.getByRole('dialog', {+ name: `Rename ${folder}`,+ exact: true,+ })+ await expect(rename).toBeVisible()+ await expect(rename.getByLabel('New name')).toHaveValue(folder)+ await rename.getByLabel('New name').fill(renamed)+ const move = page.waitForResponse(+ r => isApiPath(r.url(), `${filesPath}/move`) && r.request().method() === 'POST'+ )+ await rename.getByRole('button', { name: 'Rename', exact: true }).click()+ expect((await move).ok()).toBe(true)+ await expect(rename).toBeHidden()+ await expect(page.getByRole('button', { name: renamed, exact: true })).toBeVisible()+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeHidden()})-- await page.getByRole('button', { name: `Rename ${folderName}`, exact: true }).click()- const renameDialog = page.getByRole('dialog', {- name: `Rename ${folderName}`,- exact: true,+ await test.step('reopen through the list and verify persisted folder state', async () => {+ await page.getByRole('button', { name: 'Back to Agent Files', exact: true }).click()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)+ const row = page.getByRole('link', {+ name: `Open shared filesystem ${name}`,+ exact: true,+ })+ await expect(row).toBeVisible()+ const listing = page.waitForResponse(+ r => isApiPath(r.url(), filesPath) && r.request().method() === 'GET'+ )+ await row.click()+ await expect(page).toHaveURL(`${CONTROL_UI_URL}${detailPath}`)+ await expect(page.getByRole('heading', { name, exact: true })).toBeVisible()+ const response = await listing+ expect(response.status()).toBe(200)+ const files = (await response.json()) as {+ data: { entries: Array<{ name: string; kind: string }> }+ }+ expect(files.data.entries).toContainEqual(+ expect.objectContaining({ name: renamed, kind: 'directory' })+ )+ expect(files.data.entries.some(entry => entry.name === folder)).toBe(false)+ await expect(page.getByRole('button', { name: renamed, exact: true })).toBeVisible()+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeHidden()+ await screenshotAndLog(page, testInfo, 'control-ui-shared-fs-folders')})- await expect(renameDialog).toBeVisible({ timeout: 20_000 })- await renameDialog.getByLabel('New name').fill(renamedFolderName)- await renameDialog.getByRole('button', { name: 'Rename', exact: true }).click()-- await expect(page.getByRole('button', { name: renamedFolderName, exact: true })).toBeVisible({- timeout: 20_000,- })- await expect(- page.getByRole('button', { name: `Rename ${folderName}`, exact: true })- ).toBeHidden({ timeout: 20_000 })-- await screenshotAndLog(page, testInfo, 'control-ui-shared-fs-folders')} finally {- await api(page.request, 'DELETE', sharedFsDeletePath(sharedFsName))+ if (createdUid) {+ // Cleanup only. DELETE has no atomic UID precondition; require an isolated lane.+ const current = await api<Filesystem>(page.request, 'GET', apiPath)+ if (current.status !== 404) {+ expect(current.status).toBe(200)+ expect(current.data.metadata?.name).toBe(name)+ expect(current.data.metadata?.uid).toBe(createdUid)+ expect(current.data.spec?.retainOnDelete).toBe(false)+ expect((await api(page.request, 'DELETE', apiPath)).status).toBe(204)+ await expect+ .poll(async () => (await api(page.request, 'GET', apiPath)).status, { timeout: 60_000 })+ .toBe(404)+ }+ }}})+ test('unauthenticated guard rejects the creation route and protected API', async ({ page }) => {+ assertAllowedTarget('CONTROL_UI_URL', CONTROL_UI_URL)+ // Explicit negative guard: this fresh test context has never signed in.+ await page.goto(`${CONTROL_UI_URL}/agent-files/new`)+ await expect(page).toHaveURL(+ url => url.pathname === '/' && url.searchParams.get('next') === '/agent-files/new'+ )+ await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()+ await expect(page.getByLabel('Username or email')).toBeVisible()+ await expect(+ page.getByRole('heading', {+ name: 'Create SharedFileSystem',+ exact: true,+ })+ ).toBeHidden()+ // Negative authorization probe only; cannot advance the happy path.+ const denied = await api(page.request, 'GET', collectionPath)+ expect(denied.status).toBe(401)+ expect(denied.data).not.toHaveProperty('items')+ })})diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 41e577c2d..72a2f6ee1 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.564",+ "version": "0.1.565","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.564",+ "version": "0.1.565","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 48f6afc2f..82f1c7427 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.564",+ "version": "0.1.565","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex 626217c80..f6e93875e 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.305",+ "version": "0.1.306","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.305",+ "version": "0.1.306","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex e3773afa1..d95e80a36 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.305",+ "version": "0.1.306","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.tsindex 9c51b6d4b..9c11b2753 100644--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts@@ -1260,7 +1260,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {coreApi: {readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(),createNamespacedServiceAccount: vi.fn(async () => ({})),- createNamespacedService: vi.fn(async () => ({})),+ createNamespacedService: vi.fn(),+ readNamespacedService: vi.fn(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/host': 'team-mission',+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ spec: { clusterIP: '10.0.0.1' },+ })),+ replaceNamespacedService: vi.fn(async ({ body }) => body),createNamespacedSecret: vi.fn(async () => ({})),replaceNamespacedSecret: vi.fn(async () => ({})),deleteNamespacedPersistentVolumeClaim: vi.fn(),@@ -1273,20 +1287,44 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {} as unknown as k8s.RbacAuthorizationV1Api,networkingApi: {createNamespacedNetworkPolicy: vi.fn(async () => ({})),- readNamespacedNetworkPolicy: vi.fn(async () => ({})),+ readNamespacedNetworkPolicy: vi.fn(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/host': 'team-mission',+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ })),replaceNamespacedNetworkPolicy: vi.fn(async () => ({})),} as unknown as k8s.NetworkingV1Api,appsApi: {- // reconcile() now creates two Deployments (per-Host channel-reader ++ // reconcile() now converges two Deployments (per-Host channel-reader +// host). Capture only the host body — the channel-reader Deployment// does not carry CLERUM_CONTEXT_FILES_MOUNTS.- createNamespacedDeployment: vi.fn(async (req: { body: k8s.V1Deployment }) => {+ createNamespacedDeployment: vi.fn(),+ replaceNamespacedDeployment: vi.fn(async (req: { body: k8s.V1Deployment }) => {if (req.body.metadata?.name === 'team-mission') {captured.body = req.body}return {}}),- readNamespacedDeployment: vi.fn(async () => ({ status: { readyReplicas: 1 } })),+ readNamespacedDeployment: vi.fn(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/host': 'team-mission',+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ status: { readyReplicas: 1 },+ })),} as unknown as k8s.AppsV1Api,})await reconciler.reconcile(makeHost())@@ -1319,7 +1357,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {coreApi: {readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(desktopHost),createNamespacedServiceAccount: vi.fn(async () => ({})),- createNamespacedService: vi.fn(async () => ({})),+ createNamespacedService: vi.fn(),+ readNamespacedService: vi.fn(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/host': 'team-mission',+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ spec: { clusterIP: '10.0.0.1' },+ })),+ replaceNamespacedService: vi.fn(async ({ body }) => body),createNamespacedSecret: vi.fn(async () => ({})),replaceNamespacedSecret: vi.fn(async () => ({})),deleteNamespacedPersistentVolumeClaim: vi.fn(),@@ -1332,8 +1384,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {} as unknown as k8s.RbacAuthorizationV1Api,networkingApi: networkingApi as unknown as k8s.NetworkingV1Api,appsApi: {- createNamespacedDeployment: vi.fn(async () => ({})),- readNamespacedDeployment: vi.fn(async () => ({ status: { readyReplicas: 1 } })),+ createNamespacedDeployment: vi.fn(),+ replaceNamespacedDeployment: vi.fn(async ({ body }) => body),+ readNamespacedDeployment: vi.fn(async ({ name, namespace }) => ({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: {+ 'clerum.io/host': 'team-mission',+ 'clerum.io/managed-by': 'host-context-controller',+ },+ },+ status: { readyReplicas: 1 },+ })),} as unknown as k8s.AppsV1Api,})diff --git a/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts b/host-context-controller/src/__tests__/hostServiceNoopGate.test.tsindex b72c4561f..af0982594 100644--- a/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts+++ b/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts@@ -47,10 +47,11 @@ describe('Host ensureService no-op gate', () => {})})- it('CREATE-SVC-1: successful create never reads or replaces', async () => {+ it('CREATE-SVC-1: absent Service is read once then created without replace', async () => {+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })await (reconciler as any).ensureService(host)expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()- expect(coreApi.readNamespacedService).not.toHaveBeenCalled()+ expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()expect(coreApi.replaceNamespacedService).not.toHaveBeenCalled()})@@ -61,6 +62,8 @@ describe('Host ensureService no-op gate', () => {const log = vi.spyOn(console, 'log')try {await (reconciler as any).ensureService(host)+ expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()+ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()expect(coreApi.replaceNamespacedService).not.toHaveBeenCalled()expect(updatedServiceLogs(log, 'Service "chatllm"')).toEqual([])} finally {diff --git a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.tsindex ef478fc3a..6add63ca3 100644--- a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts+++ b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts@@ -201,6 +201,9 @@ describe('llm-hooks effective pod egress (N5)', () => {// Positive control: without this, the two assertions above would also pass if// effectiveEgress() were simply blind to egress rules.it('does surface the egress a declaring hook is granted (scoped DNS + its target)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const dialer = makeHook('dialer', {target: {image: { ref: IMG, port: 8080, egressBindings: [{ cidr: '8.8.8.8/32', ports: [443] }] },diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.tsindex f4e15bf35..027b952dc 100644--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts@@ -57,8 +57,11 @@ function stored(value: string): string {}function createdDeployment(appsApi: ReturnType<typeof createMockAppsApi>): k8s.V1Deployment {- const call = appsApi.createNamespacedDeployment.mock.calls[0]- return (call[0] as { body: k8s.V1Deployment }).body+ const call =+ appsApi.replaceNamespacedDeployment.mock.calls.at(-1) ??+ appsApi.createNamespacedDeployment.mock.calls.at(-1)+ expect(call, 'the reconcile must write a real desired Deployment').toBeDefined()+ return (call![0] as { body: k8s.V1Deployment }).body}function credentialsRevisionOf(deployment: k8s.V1Deployment): string | undefined {@@ -92,9 +95,17 @@ function deploymentReadyWrites(customApi: ReturnType<typeof createMockCustomApi>}/** A Deployment whose rollout has fully converged. */-function convergedDeployment(name?: string) {+function convergedDeployment(name = 'linear') {return {- metadata: { name, resourceVersion: '1', generation: 2, labels: {} },+ metadata: {+ name,+ resourceVersion: '1',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -114,6 +125,23 @@ describe('connector credentials revision (issue #223)', () => {beforeEach(() => {vi.clearAllMocks()+ let live: k8s.V1Deployment | undefined+ // This suite creates a connector, then observes the same API object on rotation.+ appsApi.readNamespacedDeployment.mockImplementation(async () => {+ if (!live) throw { code: 404 }+ return structuredClone(live)+ })+ const persist = async ({ body }: { body: k8s.V1Deployment }) => {+ live = {+ ...structuredClone(body),+ metadata: { ...body.metadata, resourceVersion: '7', generation: 1 },+ status: { observedGeneration: 1, replicas: 1, updatedReplicas: 1, readyReplicas: 1 },+ }+ return live+ }+ appsApi.createNamespacedDeployment.mockImplementation(persist)+ appsApi.replaceNamespacedDeployment.mockImplementation(persist)+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })coreApi.readNamespacedSecret.mockResolvedValue({metadata: { name: 'linear-credentials' },data: { 'api-key': stored('old-key') },@@ -140,7 +168,7 @@ describe('connector credentials revision (issue #223)', () => {await reconciler.reconcile(makeServer())const first = credentialsRevisionOf(createdDeployment(appsApi))- vi.clearAllMocks()+ appsApi.replaceNamespacedDeployment.mockClear()coreApi.readNamespacedSecret.mockResolvedValue({data: { 'webhook-secret': stored('k2'), 'api-key': stored('k1') },})@@ -156,7 +184,7 @@ describe('connector credentials revision (issue #223)', () => {await reconciler.reconcile(makeServer())const before = credentialsRevisionOf(createdDeployment(appsApi))- vi.clearAllMocks()+ appsApi.replaceNamespacedDeployment.mockClear()coreApi.readNamespacedSecret.mockResolvedValue({data: { 'api-key': stored('rotated-key') },})@@ -183,7 +211,15 @@ describe('connector credentials revision (issue #223)', () => {Object.assign(new Error('already exists'), { code: 409 }))appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { name: 'linear', resourceVersion: '7', generation: 1, labels: {} },+ metadata: {+ name: 'linear',+ resourceVersion: '7',+ generation: 1,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: {replicas: 1,template: {@@ -219,6 +255,9 @@ describe('generation-aware rollout readiness (issue #223)', () => {beforeEach(() => {vi.clearAllMocks()+ // Initial create reads are absent; preserve the queued rollout observations after it.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,@@ -230,7 +269,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {it('is NOT ready while the controller has not observed the current generation', async () => {appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 3, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 3,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },// The old pod is still Ready — the trap the pre-#223 check fell into.status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 1 },@@ -243,7 +292,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {it('is NOT ready while replicas are not yet updated', async () => {appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 2 },status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 2 },})@@ -255,7 +314,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {it('is NOT ready while a replica is unavailable', async () => {appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -288,7 +357,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {// is exactly the e2e E2 scenario; without `totalReplicas === updatedReplicas`// the poll reports the failed rotation as healthy.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -321,6 +400,9 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',beforeEach(() => {vi.clearAllMocks()+ // Initial create reads are absent; preserve the queued rollout observations after it.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })vi.useFakeTimers()coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {@@ -340,7 +422,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',// converged by the time the poll runs.appsApi.readNamespacedDeployment.mockResolvedValueOnce({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },})@@ -355,7 +447,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',it('converges from the immediate False to a later True, instead of staying False', async () => {appsApi.readNamespacedDeployment.mockResolvedValueOnce({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },})@@ -375,7 +477,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',it('gives up loudly on the CRD, with the rollout numbers, when it never converges', async () => {appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -402,7 +514,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',// behind, so the rollout never converges (total 2 vs updated 1) — the// exact E2 shape. Deployment generation never changes: same rollout.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -437,7 +559,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',it('grants a NEW rotation (bumped Deployment generation) a fresh poll budget mid-window', async () => {const notConverged = (generation: number) => ({- metadata: { generation, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: generation,@@ -478,7 +610,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',// Deployment watch — so the CRD would keep the OLD generation's// RolloutIncomplete forever even though the NEW rollout converges.const notConverged = (generation: number) => ({- metadata: { generation, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: generation,@@ -490,7 +632,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',},})const converged = (generation: number) => ({- metadata: { generation, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: generation,@@ -564,7 +716,17 @@ describe('poll-window fence survives a superseding reconcile (F1, watch-recovery// A rollout mid-flight (total 2 vs updated 1) that never converges on its own.const midRollout = {- metadata: { generation: 2, labels: {} },+ metadata: {+ name: 'linear',+ namespace: 'mcp-server',+ resourceVersion: '1',+ uid: 'linear-deployment',+ generation: 2,+ labels: {+ 'clerum.io/managed-by': 'host-context-controller',+ 'clerum.io/mcpserver': 'linear',+ },+ },spec: { replicas: 1 },status: {observedGeneration: 2,@@ -578,6 +740,9 @@ describe('poll-window fence survives a superseding reconcile (F1, watch-recoverybeforeEach(() => {vi.clearAllMocks()+ // Initial create reads are absent; preserve the queued rollout observations after it.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })vi.useFakeTimers()authorityGeneration = 1currentServer = makeServer()diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.tsindex fa30ec7a5..675ac83ad 100644--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts@@ -88,6 +88,9 @@ describe('McpServer generated pod hardening', () => {beforeEach(() => {vi.clearAllMocks()+ // These tests inspect the first generated workload, before it exists in the API.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.tsindex 044452efd..833e78037 100644--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts@@ -119,15 +119,15 @@ function mockHccOwnedRuntimeReads(}appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1', labels },+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },status: { readyReplicas: 1 },})coreApi.readNamespacedConfigMap.mockResolvedValue({- metadata: { resourceVersion: '1', labels },+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },data: {},})coreApi.readNamespacedService.mockResolvedValue({- metadata: { resourceVersion: '1', labels },+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },spec: { clusterIP: '10.0.0.1' },})}@@ -719,6 +719,9 @@ describe('McpServerReconciler remote egress proxy', () => {describe('reconcile call order for remote servers', () => {it('should create ConfigMap before Deployment', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const callOrder: string[] = []coreApi.createNamespacedConfigMap.mockImplementation(async () => {@@ -745,6 +748,9 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should use the platform image without rewriting a divergent remote desired image', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const callOrder: string[] = []const staleRemote = cloneServer(REMOTE_SERVER, {image: LEGACY_REMOTE_EGRESS_IMAGE,@@ -807,6 +813,9 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should also create Deployment and Service', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })await reconciler.reconcile(REMOTE_SERVER)expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()@@ -836,6 +845,9 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should still create Deployment and Service', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })await reconciler.reconcile(LOCAL_SERVER)expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()@@ -907,7 +919,12 @@ describe('McpServerReconciler remote egress proxy', () => {it('should replace an existing remote egress proxy Deployment with the current image', async () => {appsApi.createNamespacedDeployment.mockRejectedValue(make409Error())appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: 'old-rv' },+ metadata: {+ name: REMOTE_SERVER.name,+ namespace: REMOTE_SERVER.namespace,+ resourceVersion: 'old-rv',+ labels: { [MANAGED_BY_LABEL]: MANAGED_BY_VALUE, [MCPSERVER_LABEL]: REMOTE_SERVER.name },+ },spec: {template: {spec: {@@ -1052,6 +1069,9 @@ describe('McpServerReconciler remote egress proxy', () => {})it('should clear status tracking after delete', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })let current: McpServerCRD | undefined = REMOTE_SERVERreconciler.setResolveCurrentServer(() => current)// First reconcile to populate status@@ -1237,6 +1257,9 @@ describe('plugin image-host allowlist (2.3)', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()const customApi = createMockCustomApi()+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const reconciler = buildReconciler(appsApi, coreApi, customApi)const server = cloneServer(LOCAL_SERVER, { image: 'docker.io/evil/x:1' })@@ -1277,6 +1300,9 @@ describe('plugin image-host allowlist (2.3)', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()const customApi = createMockCustomApi()+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const reconciler = buildReconciler(appsApi, coreApi, customApi)const server = cloneServer(LOCAL_SERVER, { image: 'registry.evenfire.ai/acme/x:1' })@@ -1292,6 +1318,9 @@ describe('plugin image-host allowlist (2.3)', () => {const appsApi = createMockAppsApi()const coreApi = createMockCoreApi()const customApi = createMockCustomApi()+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const reconciler = buildReconciler(appsApi, coreApi, customApi)const cfg = (await import('../config')).config as { enforcePluginImageAllowlist: boolean }cfg.enforcePluginImageAllowlist = truediff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.tsindex a9dd07df6..fb12036ce 100644--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts@@ -90,6 +90,9 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {beforeEach(() => {vi.clearAllMocks()+ // These tests inspect the first generated workload, before it exists in the API.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })reconciler = new McpServerReconciler({} as k8s.KubeConfig, {assumeInventoryAuthorityWhenUnconfigured: true,appsApi: asAppsApi(appsApi),diff --git a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.tsindex c3f093563..6c82c70fb 100644--- a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts+++ b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts@@ -1,9 +1,12 @@import { beforeEach, describe, expect, it, vi } from 'vitest'+import type * as k8s from '@kubernetes/client-node'import {type SharedFileSystemFactoryConfig,WFC_MOUNT_PATH,+ buildDeployment,buildEgressNetworkPolicy,buildIngressNetworkPolicy,+ buildService,pvcName,wfcDeploymentName,wfcEgressPolicyName,@@ -11,6 +14,7 @@ import {wfcInitJobNamePrefix,wfcServiceName,} from '../k8s/sharedFileSystemFactory'+import { createsTotal, existenceReadsTotal } from '../metrics'import { SharedFileSystemReconciler } from '../sharedFileSystemReconciler'import type { SharedFileSystemCRD } from '../types'@@ -52,6 +56,14 @@ function err(code: number, msg = 'k8s error'): Error & { code: number } {const notFound = () => err(404, 'not found')const alreadyExists = () => err(409, 'already exists')+function liveDeployment(+ sfs = makeSfs(),+ status: k8s.V1DeploymentStatus = { readyReplicas: 1, availableReplicas: 1 }+): k8s.V1Deployment {+ const desired = buildDeployment(sfs, factoryConfig)+ return { ...desired, metadata: { ...desired.metadata, resourceVersion: '1' }, status }+}+function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {// Status/cleanup tests use already-converged policies with real identity and// ownership. Cold-start tests explicitly arrange absence for their own reads.@@ -61,11 +73,15 @@ function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {])const appsApi = {createNamespacedDeployment: vi.fn().mockResolvedValue({}),- // Truthful status (#592) reads readyReplicas; conflict-retry reads metadata.- readNamespacedDeployment: vi.fn().mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 1, availableReplicas: 1 },- }),+ // Existence and readiness share an actual named Deployment; status tests+ // vary its status without erasing its desired spec or ownership metadata.+ readNamespacedDeployment: vi.fn(+ async ({ name }: { name: string }): Promise<k8s.V1Deployment> => {+ const sfs = systems.find(item => wfcDeploymentName(item) === name)+ if (!sfs) throw new Error(`Missing Deployment fixture for ${name}`)+ return liveDeployment(sfs)+ }+ ),replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),deleteNamespacedDeployment: vi.fn().mockResolvedValue({}),}@@ -73,6 +89,17 @@ function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),createNamespacedService: vi.fn().mockResolvedValue({}),+ readNamespacedService: vi.fn(async ({ name }: { name: string }): Promise<k8s.V1Service> => {+ const sfs = systems.find(item => wfcServiceName(item) === name)+ if (!sfs) throw new Error(`Missing Service fixture for ${name}`)+ const desired = buildService(sfs, factoryConfig)+ return {+ ...desired,+ metadata: { ...desired.metadata, resourceVersion: '1' },+ spec: { ...desired.spec, clusterIP: '10.96.1.23' },+ }+ }),+ replaceNamespacedService: vi.fn(),deleteNamespacedService: vi.fn().mockResolvedValue({}),// Truthful status (#592): assessReadiness reads the PVC bind state, and// detectWfcWedge lists the wfc Pod. Defaults = happy path (PVC Bound, no@@ -152,16 +179,16 @@ describe('SharedFileSystemReconciler — monotonic status on re-reconcile (#592phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)return {}})- mocks.coreApi.createNamespacedService.mockImplementation(async () => {+ mocks.coreApi.readNamespacedService.mockImplementation(async () => {phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)- return {}+ return buildService(sfs, factoryConfig)})// Re-reconcile the already-Ready SFS (mirrors a Context-change re-reconcile).await reconciler.reconcile(sfs)expect(reconciler.getStatus(sfs).phase).toBe('Ready')- expect(phasesSeenMidReconcile.length).toBeGreaterThan(0)+ expect(phasesSeenMidReconcile).toHaveLength(2)for (const phase of phasesSeenMidReconcile) {expect(phase).toBe('Ready')}@@ -179,6 +206,8 @@ describe('SharedFileSystemReconciler — happy path', () => {it('creates PVC, Service, Deployment, and both NetworkPolicies — and NEVER an init Job', async () => {const sfs = makeSfs()+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())+ mocks.coreApi.readNamespacedService.mockRejectedValueOnce(notFound())mocks.networkingApi.readNamespacedNetworkPolicy.mockRejectedValueOnce(notFound()).mockRejectedValueOnce(notFound())@@ -201,6 +230,7 @@ describe('SharedFileSystemReconciler — happy path', () => {it('builds the Deployment with a root initContainer that seeds the PVC in the same pod', async () => {const sfs = makeSfs()+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())await reconciler.reconcile(sfs)const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].bodyconst podSpec = dep.spec.template.spec@@ -271,9 +301,13 @@ describe('SharedFileSystemReconciler — legacy init-Job cleanup', () => {it('is non-fatal: a cleanup list failure does NOT block the Deployment rewrite', async () => {const mocks = makeMocks()mocks.batchApi.listNamespacedJob.mockRejectedValue(err(500, 'list boom'))+ const drifted = liveDeployment()+ drifted.spec!.replicas = 9+ mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce(drifted)const reconciler = makeReconciler(mocks)await expect(reconciler.reconcile(makeSfs())).resolves.toBeUndefined()- expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()expect(reconciler.getStatus(makeSfs()).phase).toBe('Ready')})@@ -335,17 +369,26 @@ describe('SharedFileSystemReconciler — idempotence', () => {it('Deployment 409 (already exists) triggers a replace via the conflict-retry helper', async () => {const mocks = makeMocks()mocks.appsApi.createNamespacedDeployment.mockRejectedValueOnce(alreadyExists())+ const drifted = liveDeployment()+ drifted.spec!.replicas = 9+ mocks.appsApi.readNamespacedDeployment+ .mockRejectedValueOnce(notFound())+ .mockResolvedValueOnce(drifted)const reconciler = makeReconciler(mocks)await reconciler.reconcile(makeSfs())- expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalled()+ expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(3)+ expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalled()})it('preserves external annotations without replacing the desired Deployment spec', async () => {const mocks = makeMocks()- mocks.appsApi.createNamespacedDeployment.mockRejectedValueOnce(alreadyExists())mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '7', annotations: { 'operator.example/keep': 'yes' } },+ metadata: {+ ...liveDeployment().metadata,+ resourceVersion: '7',+ annotations: { 'operator.example/keep': 'yes' },+ },spec: {replicas: 9,strategy: { type: 'RollingUpdate' },@@ -374,26 +417,27 @@ describe('SharedFileSystemReconciler — idempotence', () => {await reconciler.reconcile(makeSfs())- const desiredBody = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body+ const desiredBody = buildDeployment(makeSfs(), factoryConfig)+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()const replaceBody = mocks.appsApi.replaceNamespacedDeployment.mock.calls[0][0].bodyexpect(replaceBody.metadata.annotations).toMatchObject({ 'operator.example/keep': 'yes' })expect(replaceBody.spec.template.metadata.annotations).toEqual({'kubectl.kubernetes.io/restartedAt': '2026-08-22T12:00:00Z','operator.example/pod-marker': 'keep',})- expect(replaceBody.spec.replicas).toBe(desiredBody.spec.replicas)- expect(replaceBody.spec.strategy).toEqual(desiredBody.spec.strategy)- expect(replaceBody.spec.selector).toEqual(desiredBody.spec.selector)+ expect(replaceBody.spec.replicas).toBe(desiredBody.spec!.replicas)+ expect(replaceBody.spec.strategy).toEqual(desiredBody.spec!.strategy)+ expect(replaceBody.spec.selector).toEqual(desiredBody.spec!.selector)expect(replaceBody.spec.template.metadata.labels).toEqual(- desiredBody.spec.template.metadata.labels+ desiredBody.spec!.template.metadata!.labels)expect(replaceBody.spec.template.spec.containers).toEqual(- desiredBody.spec.template.spec.containers+ desiredBody.spec!.template.spec!.containers)expect(replaceBody.spec.template.spec.initContainers).toEqual(- desiredBody.spec.template.spec.initContainers+ desiredBody.spec!.template.spec!.initContainers)- expect(replaceBody.spec.template.spec).toEqual(desiredBody.spec.template.spec)+ expect(replaceBody.spec.template.spec).toEqual(desiredBody.spec!.template.spec)})it('never creates a Job across repeated reconciles', async () => {@@ -529,6 +573,7 @@ describe('SharedFileSystemReconciler — fullReconcile', () => {describe('SharedFileSystemReconciler — wfc env wiring (sanity)', () => {it('passes WSF_MOUNT_PATH=/workspace and SFS identity into the Deployment env', async () => {const mocks = makeMocks()+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())const reconciler = makeReconciler(mocks)await reconciler.reconcile(makeSfs())const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body@@ -555,10 +600,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod not created yetconst sfs = makeSfs()await reconciler.reconcile(sfs)@@ -569,10 +613,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({items: [{@@ -607,10 +650,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({items: [{@@ -640,10 +682,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Bound' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({items: [{@@ -672,10 +713,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))await reconciler.reconcile(sfs)expect(reconciler.getStatus(sfs).phase).toBe('Initializing')// Second reconcile falls through to the default happy-path mocks.@@ -748,10 +788,9 @@ describe('SharedFileSystemReconciler — status-write dirty check + lastTransitimocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))await reconciler.reconcile(sfs)expect(reconciler.getStatus(sfs).phase).toBe('Initializing')const writesAfterFirst = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length@@ -794,10 +833,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readinemocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Bound', capacity: { storage: '5Gi' } },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod restartingconst sfs = makeSfs()await reconciler.reconcile(sfs)@@ -809,10 +847,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readinemocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Bound', capacity: { storage: '5Gi' } },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({items: [{@@ -837,10 +874,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readinemocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({status: { phase: 'Pending' },})- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({- metadata: { resourceVersion: '1' },- status: { readyReplicas: 0 },- })+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(+ liveDeployment(makeSfs(), { readyReplicas: 0 })+ )mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] })const sfs = makeSfs()await reconciler.reconcile(sfs)@@ -859,3 +895,153 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readineexpect(reconciler.isMountable(sfs)).toBe(false)})})++// Exercise the actual resource adapters without readiness reads masking their+// physical GET counts. Integration status and recovery remain covered above.+describe.each(['Deployment', 'Service'] as const)('SFS %s read-first contract', kind => {+ beforeEach(() => {+ createsTotal.reset()+ existenceReadsTotal.reset()+ })+ function setup() {+ const mocks = makeMocks()+ const reconciler = makeReconciler(mocks) as unknown as {+ ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>+ ensureService(sfs: SharedFileSystemCRD): Promise<string>+ }+ const sfs = makeSfs()+ const name = kind === 'Deployment' ? wfcDeploymentName(sfs) : wfcServiceName(sfs)+ const read =+ kind === 'Deployment'+ ? mocks.appsApi.readNamespacedDeployment+ : mocks.coreApi.readNamespacedService+ const create =+ kind === 'Deployment'+ ? mocks.appsApi.createNamespacedDeployment+ : mocks.coreApi.createNamespacedService+ const replace =+ kind === 'Deployment'+ ? mocks.appsApi.replaceNamespacedDeployment+ : mocks.coreApi.replaceNamespacedService+ const run = () =>+ kind === 'Deployment' ? reconciler.ensureDeployment(sfs) : reconciler.ensureService(sfs)+ return { mocks, sfs, name, read, create, replace, run }+ }+ async function count(metric: typeof createsTotal, outcome: string) {+ return (+ (await metric.get()).values.find(+ row => row.labels.kind === kind && row.labels.outcome === outcome+ )?.value ?? 0+ )+ }++ it('returns the existing name with one GET and no POST or PUT', async () => {+ const h = setup()+ await expect(h.run()).resolves.toBe(h.name)+ expect(h.read).toHaveBeenCalledExactlyOnceWith({ name: h.name, namespace: 'mcp-host' })+ expect(h.create).not.toHaveBeenCalled()+ expect(h.replace).not.toHaveBeenCalled()+ expect(await count(createsTotal, 'skipped')).toBe(1)+ expect(await count(existenceReadsTotal, 'found')).toBe(1)+ })++ it('creates only after GET404 and returns the created name', async () => {+ const h = setup()+ h.read.mockRejectedValueOnce(notFound())+ await expect(h.run()).resolves.toBe(h.name)+ expect(h.read).toHaveBeenCalledTimes(1)+ expect(h.create).toHaveBeenCalledExactlyOnceWith({+ namespace: 'mcp-host',+ body:+ kind === 'Deployment'+ ? buildDeployment(h.sfs, factoryConfig)+ : buildService(h.sfs, factoryConfig),+ })+ expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])+ expect(h.replace).not.toHaveBeenCalled()+ expect(await count(createsTotal, 'created')).toBe(1)+ expect(await count(createsTotal, 'skipped')).toBe(0)+ })++ it('consumes a fresh GET after POST409 rather than reusing absence', async () => {+ const h = setup()+ h.read.mockRejectedValueOnce(notFound())+ h.create.mockRejectedValueOnce(alreadyExists())+ await expect(h.run()).resolves.toBe(h.name)+ expect(h.read).toHaveBeenCalledTimes(2)+ expect(h.create).toHaveBeenCalledTimes(1)+ expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])+ expect(h.create.mock.invocationCallOrder[0]).toBeLessThan(h.read.mock.invocationCallOrder[1])+ expect(h.replace).not.toHaveBeenCalled()+ expect(await count(createsTotal, 'conflict')).toBe(1)+ expect(await count(createsTotal, 'skipped')).toBe(0)+ })++ it.each([403, 500])('propagates GET%s without any writes or skip', async code => {+ const h = setup()+ const failure = err(code)+ h.read.mockRejectedValueOnce(failure)+ await expect(h.run()).rejects.toBe(failure)+ expect(h.read).toHaveBeenCalledTimes(1)+ expect(h.create).not.toHaveBeenCalled()+ expect(h.replace).not.toHaveBeenCalled()+ expect(await count(createsTotal, 'skipped')).toBe(0)+ expect(await count(existenceReadsTotal, 'error')).toBe(1)+ })++ it('propagates fresh GET403 after POST409', async () => {+ const h = setup()+ const failure = err(403)+ h.read.mockRejectedValueOnce(notFound()).mockRejectedValueOnce(failure)+ h.create.mockRejectedValueOnce(alreadyExists())+ await expect(h.run()).rejects.toBe(failure)+ expect(h.read).toHaveBeenCalledTimes(2)+ expect(h.create).toHaveBeenCalledTimes(1)+ expect(h.replace).not.toHaveBeenCalled()+ expect(await count(createsTotal, 'skipped')).toBe(0)+ })+})++describe('SFS preserve and retry contracts', () => {+ it('keeps an existing Service unchanged even when its fields differ from desired', async () => {+ const mocks = makeMocks()+ const sfs = makeSfs()+ const service = buildService(sfs, factoryConfig)+ service.spec = { ...service.spec, clusterIP: '10.96.1.45', ports: [{ port: 9090 }] }+ mocks.coreApi.readNamespacedService.mockResolvedValueOnce(service)+ const reconciler = makeReconciler(mocks) as unknown as {+ ensureService(sfs: SharedFileSystemCRD): Promise<string>+ }+ await expect(reconciler.ensureService(sfs)).resolves.toBe(wfcServiceName(sfs))+ expect(mocks.coreApi.readNamespacedService).toHaveBeenCalledTimes(1)+ expect(mocks.coreApi.createNamespacedService).not.toHaveBeenCalled()+ expect(mocks.coreApi.replaceNamespacedService).not.toHaveBeenCalled()+ expect(service.spec.clusterIP).toBe('10.96.1.45')+ expect(service.spec.ports).toEqual([{ port: 9090 }])+ })++ it('reuses the first Deployment snapshot once and rereads its version after PUT409', async () => {+ const mocks = makeMocks()+ const sfs = makeSfs()+ let version = 0+ mocks.appsApi.readNamespacedDeployment.mockImplementation(async () => {+ const live = liveDeployment(sfs)+ live.metadata!.resourceVersion = String(++version)+ live.spec!.replicas = 9+ return live+ })+ mocks.appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(alreadyExists())+ const reconciler = makeReconciler(mocks) as unknown as {+ ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>+ }+ await expect(reconciler.ensureDeployment(sfs)).resolves.toBe(wfcDeploymentName(sfs))+ expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(+ mocks.appsApi.replaceNamespacedDeployment.mock.calls.map(+ ([{ body }]) => body.metadata.resourceVersion+ )+ ).toEqual(['1', '2'])+ })+})diff --git a/host-context-controller/src/existenceReadMetrics.test.ts b/host-context-controller/src/existenceReadMetrics.test.tsindex 8992d1c78..381ad4c23 100644--- a/host-context-controller/src/existenceReadMetrics.test.ts+++ b/host-context-controller/src/existenceReadMetrics.test.ts@@ -182,10 +182,10 @@ function readProductionSources(): Record<string, string> {function assertReadInventory(sources: Record<string, string>): void {const expectedWrapped: Record<string, number> = {'utils.ts': 1,- 'hostReconciler.ts': 8,+ 'hostReconciler.ts': 7,'reconciler.ts': 3,'llmHookReconciler.ts': 3,- 'sharedFileSystemReconciler.ts': 1,+ 'sharedFileSystemReconciler.ts': 2,'k8s/gfsK8sApi.ts': 5,'networkPolicyReconciler.ts': 2,}diff --git a/host-context-controller/src/hostReconciler.readFirst.test.ts b/host-context-controller/src/hostReconciler.readFirst.test.tsnew file mode 100644index 000000000..e6f443794--- /dev/null+++ b/host-context-controller/src/hostReconciler.readFirst.test.ts@@ -0,0 +1,239 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'+import type * as k8s from '@kubernetes/client-node'+import {+ asAppsApi,+ asCoreApi,+ asCustomApi,+ asNetworkingApi,+ asRbacApi,+ createMockAppsApi,+ createMockCoreApi,+ createMockCustomApi,+ createMockNetworkingApi,+ createMockRbacApi,+ makeStubKc,+} from '../test/__fixtures__/testMocks'+import { asApiserverService } from './__tests__/asApiserverService'+import { HostReconciler } from './hostReconciler'+import { CREATE_KINDS, createsTotal } from './metrics'+import type { HostCRD } from './types'++const host: HostCRD = {+ name: 'read-first-host',+ namespace: 'mcp-host',+ uid: 'host-uid',+ spec: { host: 'read-first-host', contextRef: 'context', secretRef: 'host-secret' },+}++function fixture() {+ const core = createMockCoreApi()+ const apps = createMockAppsApi()+ const reconciler = new HostReconciler(makeStubKc(), {+ coreApi: asCoreApi(core),+ appsApi: asAppsApi(apps),+ networkingApi: asNetworkingApi(createMockNetworkingApi()),+ rbacApi: asRbacApi(createMockRbacApi()),+ customApi: asCustomApi(createMockCustomApi()),+ })+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue('revision')+ return { core, apps, reconciler }+}++async function count(kind: string, outcome: string) {+ const sample = (await createsTotal.get()).values.find(+ v => v.labels.kind === kind && v.labels.outcome === outcome+ )+ expect(sample).toBeDefined()+ return sample!.value+}++const cases = [+ ['Host Service', 'Service', 'ensureService'],+ ['channel-reader Service', 'Service', 'reconcileChannelReaderService'],+ ['Host Deployment', 'Deployment', 'ensureDeployment'],+ ['channel-reader Deployment', 'Deployment', 'reconcileChannelReaderDeployment'],+] as const++function invoke(reconciler: HostReconciler, method: string, revalidate?: () => void) {+ return method === 'ensureDeployment'+ ? (reconciler as any)[method](host, [], 'runtime-revision', undefined, undefined, revalidate)+ : (reconciler as any)[method](host, revalidate)+}++describe('Host read-first Service and Deployment contracts', () => {+ beforeEach(() => {+ createsTotal.reset()+ for (const kind of CREATE_KINDS)+ for (const outcome of ['created', 'conflict', 'error', 'skipped'])+ createsTotal.inc({ kind, outcome }, 0)+ })++ it.each(cases)(+ '%s propagates initial GET403 without a mutation or skip',+ async (_label, kind, method) => {+ const { core, apps, reconciler } = fixture()+ const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment+ const create =+ kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment+ const replace =+ kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment+ const denied = { code: 403 }+ read.mockRejectedValueOnce(denied)+ await expect(invoke(reconciler, method)).rejects.toBe(denied)+ expect(read).toHaveBeenCalledOnce()+ expect(create).not.toHaveBeenCalled()+ expect(replace).not.toHaveBeenCalled()+ expect(await count(kind, 'skipped')).toBe(0)+ expect(await count(kind, 'error')).toBe(0)+ }+ )++ it.each(cases)(+ '%s revalidates after an absent GET before any POST',+ async (_label, kind, method) => {+ const { core, apps, reconciler } = fixture()+ const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment+ const create =+ kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment+ const replace =+ kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment+ let current = true+ const superseded = new Error('admission retired during existence GET')+ const revalidate = vi.fn(() => {+ if (!current) throw superseded+ })+ read.mockImplementationOnce(async () => {+ current = false+ throw { code: 404 }+ })+ await expect(invoke(reconciler, method, revalidate)).rejects.toBe(superseded)+ expect(read).toHaveBeenCalledOnce()+ expect(revalidate).toHaveBeenCalledTimes(2)+ expect(create).not.toHaveBeenCalled()+ expect(replace).not.toHaveBeenCalled()+ expect(await count(kind, 'skipped')).toBe(0)+ }+ )++ it('rechecks Host admission after asynchronous Deployment body resolution', async () => {+ const { apps, reconciler } = fixture()+ let current = true+ const superseded = new Error('admission retired during body resolution')+ const revalidate = vi.fn(() => {+ if (!current) throw superseded+ })+ apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ const resolveState = vi.fn(async () => {+ expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()+ current = false+ return {+ runtimeTokenRevision: 'fresh-revision',+ lifecycle: { stateless: false, state: 'active' },+ }+ })+ await expect(+ (reconciler as any).ensureDeployment(+ host,+ [],+ 'old-revision',+ undefined,+ resolveState,+ revalidate+ )+ ).rejects.toBe(superseded)+ expect(resolveState).toHaveBeenCalledOnce()+ expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(apps.replaceNamespacedDeployment).not.toHaveBeenCalled()+ expect(await count('Deployment', 'skipped')).toBe(0)+ })++ it('uses state resolved after GET404 in the first Deployment POST', async () => {+ const { apps, reconciler } = fixture()+ apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ const resolveState = vi.fn(async () => {+ expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()+ return {+ runtimeTokenRevision: 'fresh-revision',+ lifecycle: { stateless: false, state: 'active' },+ }+ })+ await (reconciler as any).ensureDeployment(host, [], 'old-revision', undefined, resolveState)+ expect(resolveState).toHaveBeenCalledOnce()+ expect(apps.createNamespacedDeployment).toHaveBeenCalledOnce()+ const body = apps.createNamespacedDeployment.mock.calls[0][0].body as k8s.V1Deployment+ expect(body.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']).toBe(+ 'fresh-revision'+ )+ expect(await count('Deployment', 'created')).toBe(1)+ expect(await count('Deployment', 'skipped')).toBe(0)+ })++ it('retains a Host Service POST failure without recording presence suppression', async () => {+ const { core, reconciler } = fixture()+ core.readNamespacedService.mockRejectedValueOnce({ code: 404 })+ core.createNamespacedService.mockRejectedValueOnce({ code: 503 })+ await expect((reconciler as any).ensureService(host)).resolves.toBeUndefined()+ expect(core.readNamespacedService).toHaveBeenCalledOnce()+ expect(core.createNamespacedService).toHaveBeenCalledOnce()+ expect(core.replaceNamespacedService).not.toHaveBeenCalled()+ expect(await count('Service', 'error')).toBe(1)+ expect(await count('Service', 'skipped')).toBe(0)+ })++ it('retains a Host Service convergence failure without recording a successful skip', async () => {+ const { core, reconciler } = fixture()+ core.replaceNamespacedService.mockRejectedValueOnce({ code: 503 })+ await expect((reconciler as any).ensureService(host)).resolves.toBeUndefined()+ expect(core.readNamespacedService).toHaveBeenCalledOnce()+ expect(core.createNamespacedService).not.toHaveBeenCalled()+ expect(core.replaceNamespacedService).toHaveBeenCalledOnce()+ expect(await count('Service', 'error')).toBe(0)+ expect(await count('Service', 'skipped')).toBe(0)+ })++ it('counts an equivalent present Host Service once without a POST or PUT', async () => {+ const { core, reconciler } = fixture()+ const desired = (reconciler as any).buildService(host) as k8s.V1Service+ core.readNamespacedService.mockResolvedValue(asApiserverService(desired))+ core.createNamespacedService.mockRejectedValue({ code: 409 })+ await (reconciler as any).ensureService(host)+ expect(core.readNamespacedService).toHaveBeenCalledOnce()+ expect(core.createNamespacedService).not.toHaveBeenCalled()+ expect(core.replaceNamespacedService).not.toHaveBeenCalled()+ expect(await count('Service', 'skipped')).toBe(1)+ })++ it('rejects an invalid channel-reader retry response before another PUT', async () => {+ const { apps, reconciler } = fixture()+ const stale = (reconciler as any).buildChannelReaderDeployment(+ host,+ 'revision'+ ) as k8s.V1Deployment+ stale.metadata = { ...stale.metadata, resourceVersion: '1' }+ stale.spec!.template.spec!.containers[0].image = 'channel-reader:stale'+ apps.readNamespacedDeployment+ .mockResolvedValueOnce(stale)+ .mockResolvedValueOnce({ metadata: { resourceVersion: '2' } })+ apps.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ await expect((reconciler as any).reconcileChannelReaderDeployment(host)).rejects.toThrow(+ 'metadata.name'+ )+ expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(apps.replaceNamespacedDeployment).toHaveBeenCalledOnce()+ expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(await count('Deployment', 'skipped')).toBe(0)+ })++ it('counts a Host Service create race as conflict and never as skipped', async () => {+ const { core, reconciler } = fixture()+ core.readNamespacedService.mockRejectedValueOnce({ code: 404 })+ core.createNamespacedService.mockRejectedValueOnce({ code: 409 })+ await (reconciler as any).ensureService(host)+ expect(core.readNamespacedService).toHaveBeenCalledTimes(2)+ expect(core.createNamespacedService).toHaveBeenCalledOnce()+ expect(core.replaceNamespacedService).toHaveBeenCalledOnce()+ expect(core.replaceNamespacedService.mock.calls[0][0].body.metadata.resourceVersion).toBe('1')+ expect(await count('Service', 'conflict')).toBe(1)+ expect(await count('Service', 'skipped')).toBe(0)+ })+})diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.tsindex e08462f3b..b185932a9 100644--- a/host-context-controller/src/hostReconciler.test.ts+++ b/host-context-controller/src/hostReconciler.test.ts@@ -1501,19 +1501,27 @@ describe('HostReconciler Host inventory mutation authority', () => {})vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {current = {...host,generation: 4,spec: { ...host.spec, contextRef: 'replacement-context' },}- return { metadata: { resourceVersion: 'rv-existing' } }+ return {+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: 'deployment-uid',+ resourceVersion: 'rv-existing',+ },+ }})await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()})@@ -1550,7 +1558,14 @@ describe('HostReconciler Host inventory mutation authority', () => {spec: { ...host.spec, contextRef: 'replacement-context' },}}- return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }+ return {+ metadata: {+ name: host.name,+ namespace: host.namespace,+ uid: 'deployment-uid',+ resourceVersion: `rv-existing-${deploymentReads}`,+ },+ }})appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })const random = vi.spyOn(Math, 'random').mockReturnValue(0)@@ -1561,7 +1576,7 @@ describe('HostReconciler Host inventory mutation authority', () => {random.mockRestore()}- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)})diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.tsindex 6fd28f36e..5f3f542a9 100644--- a/host-context-controller/src/hostReconciler.ts+++ b/host-context-controller/src/hostReconciler.ts@@ -39,6 +39,7 @@ import {import { hccLogger } from './logger'import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'import {+ createsTotal,hostCleanupDeferredTotal,hostDeleteCleanupTotal,hostFleetBenignSupersessionsTotal,@@ -77,6 +78,7 @@ import {canonicalStringify,canonicalizeValue,deploymentMatchesDesired,+ ensureResource,getErrorCode,observeCreate,observeExistenceRead,@@ -2496,50 +2498,40 @@ export class HostReconciler {}}- private async reconcileChannelReaderService(host: HostCRD): Promise<void> {+ private async reconcileChannelReaderService(+ host: HostCRD,+ revalidate?: () => void+ ): Promise<void> {const name = `channel-reader-${host.name}`const ns = config.channelsNamespaceconst service = this.buildChannelReaderService(host)-- try {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: ns,- body: service,- })- )- console.log(`[HostReconciler] Created channel-reader Service "${name}"`)- } catch (err) {- if (getErrorCode(err) !== 409) {- console.error(`[HostReconciler] Failed to create channel-reader Service "${name}":`, err)- throw err- }- try {- await replaceWithConflictRetry({+ const mutationAllowed = () => {+ revalidate?.()+ return true+ }+ await ensureResource({+ mutationAllowed,+ read: () =>+ observeExistenceRead('Service', () =>+ this.coreApi.readNamespacedService({ name, namespace: ns })+ ),+ create: () =>+ observeCreate('Service', () =>+ this.coreApi.createNamespacedService({ namespace: ns, body: service })+ ),+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),+ converge: read =>+ replaceWithConflictRetry({description: `channel-reader Service "${name}"`,logPrefix: '[HostReconciler]',body: service,mergeExisting: preserveServiceAssignedFields,isUpToDate: serviceMatchesDesired,- read: () =>- observeExistenceRead('Service', () =>- this.coreApi.readNamespacedService({ name, namespace: ns })- ),- replace: body =>- this.coreApi.replaceNamespacedService({- name,- namespace: ns,- body,- }),- })- } catch (replaceErr) {- console.error(- `[HostReconciler] Failed to update channel-reader Service "${name}":`,- replaceErr- )- throw replaceErr- }- }+ mutationAllowed,+ read,+ replace: body => this.coreApi.replaceNamespacedService({ name, namespace: ns, body }),+ }),+ })}/**@@ -2550,73 +2542,62 @@ export class HostReconciler {* clerum.io/credentials-revision pod template annotation, so the initial* pod boots with the right hash and doesn't get rolled twice when* SecretInformer fires later.- * - On 409 from create, compares a fresh, server-normalized Deployment and+ * - Reads before creation and compares a server-normalized Deployment; it* replaces only meaningful drift. Conflict retries rebuild against the* latest live replica count. A Deployment owned by a different host is* never overwritten: the initial read skips, and a retry-time ownership* change throws so write_skips_total does not count the refusal as a* healthy no-op.*/- private async reconcileChannelReaderDeployment(host: HostCRD): Promise<void> {+ private async reconcileChannelReaderDeployment(+ host: HostCRD,+ revalidate?: () => void+ ): Promise<void> {const name = `channel-reader-${host.name}`const ns = config.channelsNamespace-const revision = await this.computeChannelReaderRevisionForHost(host.name)const desired = this.buildChannelReaderDeployment(host, revision)-- try {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })- )- console.log(`[HostReconciler] Created channel-reader Deployment "${name}"`)- } catch (err) {- if (getErrorCode(err) !== 409) {- console.error(`[HostReconciler] Failed to create channel-reader "${name}":`, err)- throw err- }- let existing: k8s.V1Deployment- try {- existing = await observeExistenceRead('Deployment', () =>+ const mutationAllowed = () => {+ revalidate?.()+ return true+ }+ await ensureResource({+ mutationAllowed,+ read: () =>+ observeExistenceRead('Deployment', () =>this.appsApi.readNamespacedDeployment({ name, namespace: ns })- )- } catch (readErr) {- if (getErrorCode(readErr) === 404) {- console.warn(- `[HostReconciler] channel-reader "${name}" disappeared after create conflict; treating the reconcile as a transient failure`- )- throw readErr+ ),+ create: () =>+ observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })+ ),+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),+ converge: async (read, observed) => {+ // A post-create-conflict read still throws on 404. Later retry reads+ // keep the existing benign-disappearance policy in the retry helper.+ const existing = observed ?? (await read())+ const ownerHost = existing.metadata?.labels?.[HOST_LABEL]+ if (ownerHost && ownerHost !== host.name) {+ log.warn('Channel-reader Deployment belongs to another Host; refusing update', { name })+ return false}- console.error(`[HostReconciler] Failed to read channel-reader "${name}":`, readErr)- throw readErr- }- const ownerHost = existing.metadata?.labels?.[HOST_LABEL]- if (ownerHost && ownerHost !== host.name) {- console.warn(- `[HostReconciler] channel-reader "${name}" owned by host="${ownerHost}", skipping`- )- return- }- // The outer read above treats 404 as a transient throw so this Host- // reconcile requeues and recreates. The helper read below is the #472- // contract: 404 means gone, return, and the next Host reconcile- // recreates. Same object, two 404 policies — accepted, not a bug.- try {+ // ensureResource already supplies a consume-once snapshot for the+ // initial presence path. Only POST409 needs its owner-check read retained.+ let snapshot: k8s.V1Deployment | undefined = observed === undefined ? existing : undefinedawait replaceWithConflictRetry({description: `channel-reader Deployment "${name}"`,logPrefix: '[HostReconciler]',- // B2: rebuild against each fresh read so an unsynced CC cache- // preserves the live replica count even after a 409 retry.body: this.buildChannelReaderDeployment(host, revision, existing.spec?.replicas),- read: () =>- observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({ name, namespace: ns })- ),- replace: body =>- this.appsApi.replaceNamespacedDeployment({- name,- namespace: ns,- body,- }),+ mutationAllowed,+ read: () => {+ if (snapshot !== undefined) {+ const first = snapshot+ snapshot = undefined+ return Promise.resolve(first)+ }+ return read()+ },+ replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace: ns, body }),mergeExisting: (_body, fresh) => {const desiredWithLiveReplicas = this.buildChannelReaderDeployment(host,@@ -2631,9 +2612,6 @@ export class HostReconciler {},validateExisting: fresh => {const freshOwnerHost = fresh.metadata?.labels?.[HOST_LABEL]- // Never replace an object whose ownership changed during a retry.- // Veto here — not via isUpToDate — so write_skips_total stays a- // genuine no-op count.if (freshOwnerHost && freshOwnerHost !== host.name) {throw new Error(`channel-reader Deployment ownership changed to host "${freshOwnerHost}"; refusing replace`@@ -2642,11 +2620,8 @@ export class HostReconciler {},isUpToDate: deploymentMatchesDesired,})- } catch (replaceErr) {- console.error(`[HostReconciler] Failed to update channel-reader "${name}":`, replaceErr)- throw replaceErr- }- }+ },+ })}/**@@ -3375,18 +3350,33 @@ export class HostReconciler {}}- private async ensureService(host: HostCRD): Promise<void> {+ private async ensureService(host: HostCRD, revalidate?: () => void): Promise<void> {const service = this.buildService(host)- try {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: host.namespace,- body: service,- })- )- console.log(`[HostReconciler] Created Service "${host.name}"`)- } catch (error) {- if (getErrorCode(error) === 409) {+ const mutationAllowed = () => {+ revalidate?.()+ return true+ }+ // Initial read errors propagate. Only the existing POST/convergence paths+ // retain their historical non-throwing failures.+ await ensureResource({+ mutationAllowed,+ read: () =>+ observeExistenceRead('Service', () =>+ this.coreApi.readNamespacedService({ namespace: host.namespace, name: host.name })+ ),+ create: async () => {+ try {+ await observeCreate('Service', () =>+ this.coreApi.createNamespacedService({ namespace: host.namespace, body: service })+ )+ } catch (error) {+ if (isBenignSupersessionError(error) || (error != null && getErrorCode(error) === 409))+ throw error+ log.error('Failed to create Host Service', { host: host.name, err: error })+ }+ },+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),+ converge: async read => {try {await replaceWithConflictRetry({description: `Service "${host.name}"`,@@ -3394,13 +3384,8 @@ export class HostReconciler {body: service,mergeExisting: preserveServiceAssignedFields,isUpToDate: serviceMatchesDesired,- read: () =>- observeExistenceRead('Service', () =>- this.coreApi.readNamespacedService({- namespace: host.namespace,- name: host.name,- })- ),+ mutationAllowed,+ read,replace: body =>this.coreApi.replaceNamespacedService({namespace: host.namespace,@@ -3408,13 +3393,13 @@ export class HostReconciler {body,}),})- } catch (updateError) {- console.error(`[HostReconciler] Failed to update Service "${host.name}":`, updateError)+ } catch (error) {+ if (isBenignSupersessionError(error)) throw error+ log.error('Failed to update Host Service', { host: host.name, err: error })+ return false}- } else {- console.error(`[HostReconciler] Failed to create Service "${host.name}":`, error)- }- }+ },+ })}private async ensureDeployment(@@ -3422,7 +3407,8 @@ export class HostReconciler {mounts: ResolvedSfsMount[],runtimeTokenRevision: string,lifecycle?: EffectiveHostLifecycle,- resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>+ resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>,+ revalidate?: () => void): Promise<void> {const buildDesiredDeployment = async (): Promise<k8s.V1Deployment> => {const state = resolveStateBeforeMutation ? await resolveStateBeforeMutation() : null@@ -3433,47 +3419,43 @@ export class HostReconciler {state?.lifecycle ?? lifecycle)}- const deployment = await buildDesiredDeployment()- try {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({- namespace: host.namespace,- body: deployment,- })- )- console.log(`[HostReconciler] Created Deployment "${host.name}"`)- } catch (error) {- if (getErrorCode(error) !== 409) {- console.error(`[HostReconciler] Failed to create Deployment "${host.name}":`, error)- throw error- }- try {- await replaceWithConflictRetry({+ const mutationAllowed = () => {+ revalidate?.()+ return true+ }+ await ensureResource({+ mutationAllowed,+ read: () =>+ observeExistenceRead('Deployment', () =>+ this.appsApi.readNamespacedDeployment({ namespace: host.namespace, name: host.name })+ ),+ create: async () => {+ // The initial GET may outlive the lifecycle or token-scope observation.+ const deployment = await buildDesiredDeployment()+ revalidate?.()+ return observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({ namespace: host.namespace, body: deployment })+ )+ },+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),+ converge: read =>+ replaceWithConflictRetry({description: `Deployment "${host.name}"`,logPrefix: '[HostReconciler]',- body: deployment,+ body: this.buildDeployment(host, mounts, runtimeTokenRevision, lifecycle),resolveBody: buildDesiredDeployment,mergeExisting: preserveHostDeploymentAnnotations,isUpToDate: deploymentMatchesDesired,- read: () =>- observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({- namespace: host.namespace,- name: host.name,- })- ),+ mutationAllowed,+ read,replace: body =>this.appsApi.replaceNamespacedDeployment({namespace: host.namespace,name: host.name,body,}),- })- } catch (updateError) {- console.error(`[HostReconciler] Failed to update Deployment "${host.name}":`, updateError)- throw updateError- }- }+ }),+ })}private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {@@ -4426,7 +4408,7 @@ export class HostReconciler {revalidateHostMutationBoundary()await this.ensurePvc(host)revalidateHostMutationBoundary()- await this.ensureService(host)+ await this.ensureService(host, revalidateHostMutationBoundary)// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and// ingress against the policies that exist when the connection is opened,@@ -4576,7 +4558,8 @@ export class HostReconciler {mounts,runtimeTokenProvision.revision,lifecycle.effective,- resolveDeploymentState+ resolveDeploymentState,+ revalidateHostMutationBoundary)revalidateHostMutationBoundary()@@ -4619,8 +4602,8 @@ export class HostReconciler {// expected error path — not just defense in depth.const channelReaderFailures: unknown[] = []for (const reconcileResource of [- () => this.reconcileChannelReaderService(host),- () => this.reconcileChannelReaderDeployment(host),+ () => this.reconcileChannelReaderService(host, revalidateHostMutationBoundary),+ () => this.reconcileChannelReaderDeployment(host, revalidateHostMutationBoundary),]) {try {revalidateHostMutationBoundary()diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.tsindex 7f9c673ee..f5e7c570a 100644--- a/host-context-controller/src/llmHookReconciler.test.ts+++ b/host-context-controller/src/llmHookReconciler.test.ts@@ -248,6 +248,9 @@ describe('LlmHookReconciler', () => {// ─── Dedup (§10) ───────────────────────────────────────────────────it('co-locates two same-pod-key hooks on ONE Deployment/Service', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const a = makeHook({name: 'a',spec: {@@ -275,6 +278,9 @@ describe('LlmHookReconciler', () => {})it('distinct pod keys → two Deployments', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const a = makeHook({name: 'a',spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },@@ -286,6 +292,9 @@ describe('LlmHookReconciler', () => {hooks.set('a', a)hooks.set('b', b)await reconciler.reconcile(a)+ // The second pod key names a different absent workload.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })await reconciler.reconcile(b)expect(new Set(deploymentNames(appsApi)).size).toBe(2)})@@ -293,6 +302,9 @@ describe('LlmHookReconciler', () => {// ─── Path collision (§10) ──────────────────────────────────────────it('fails the path-collision loser closed while serving the winner', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const a = makeHook({name: 'a',spec: {@@ -427,6 +439,9 @@ describe('LlmHookReconciler', () => {// ─── Concurrency (§10) ─────────────────────────────────────────────it('serializes concurrent reconciles of two CRs sharing a pod key', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })let inFlight = 0let maxInFlight = 0appsApi.createNamespacedDeployment.mockImplementation(async () => {@@ -477,9 +492,13 @@ describe('LlmHookReconciler', () => {labels: { [MANAGED_BY_LABEL]: MANAGED_BY_VALUE, [HOOK_PODKEY_LABEL]: oldKey },},}- appsApi.readNamespacedDeployment.mockResolvedValue(ownedOld)- coreApi.readNamespacedService.mockResolvedValue(ownedOld)- networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(ownedOld)+ const readOld = async ({ name }: { name: string }) => {+ if (name !== podKeyResourceName(oldKey)) throw { code: 404 }+ return { ...ownedOld, metadata: { ...ownedOld.metadata, name } }+ }+ appsApi.readNamespacedDeployment.mockImplementation(readOld)+ coreApi.readNamespacedService.mockImplementation(readOld)+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(readOld)await reconciler.reconcile(newHook, oldKey)@@ -494,6 +513,9 @@ describe('LlmHookReconciler', () => {// ─── NetworkPolicy ingress reverse-index (§10) ─────────────────────it('NetworkPolicy ingress admits exactly the referencing Hosts', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// This scenario starts without a policy; the API reports absence explicitly.networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const a = makeHook({@@ -519,6 +541,9 @@ describe('LlmHookReconciler', () => {})it('co-located members: NetworkPolicy label values are comma-free (member list is an annotation)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// This scenario starts without a policy; the API reports absence explicitly.networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })// Two hooks with the SAME image target share one pod key → one shared NP.@@ -760,6 +785,9 @@ describe('LlmHookReconciler', () => {})it('hardens the hook pod: non-root, read-only root fs, dropped caps, no SA token (N8)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const a = makeHook({ name: 'a' })hooks.set('a', a)await reconciler.reconcile(a)@@ -798,6 +826,9 @@ describe('LlmHookReconciler', () => {})it('grants scoped CoreDNS egress only when a hook declares egressBindings (N5)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// This scenario starts without a policy; the API reports absence explicitly.networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const withEgress = makeHook({@@ -835,6 +866,9 @@ describe('LlmHookReconciler', () => {// "grants NO egress (not even DNS)" is what let namespace-wide DNS through// while the suite stayed green.it('declares no Egress in the per-pod-key policy for a pure responder (no egressBindings)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// This scenario starts without a policy; the API reports absence explicitly.networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const responder = makeHook({ name: 'resp' }) // default image, no egressBindings@@ -849,6 +883,9 @@ describe('LlmHookReconciler', () => {})it('allows a valid public-CIDR egress binding as an Egress NetworkPolicy rule', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// This scenario starts without a policy; the API reports absence explicitly.networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })const a = makeHook({@@ -907,6 +944,9 @@ describe('LlmHookReconciler', () => {})it('writes conditions[] + observedDigest + readyReplicas for a ready image hook', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const digest = 'sha256:' + 'b'.repeat(64)coreApi.listNamespacedPod.mockResolvedValue({items: [diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.tsindex 54e4ceb78..ecaa605de 100644--- a/host-context-controller/src/llmHookReconciler.ts+++ b/host-context-controller/src/llmHookReconciler.ts@@ -750,7 +750,7 @@ export class LlmHookReconciler {}}- // ─── Apply helpers (create-then-409 replaceWithConflictRetry) ─────────+ // ─── Read-first resource convergence ───────────────────────────────private async ensureDeployment(podKey: string,@@ -759,64 +759,86 @@ export class LlmHookReconciler {): Promise<void> {const deployment = this.buildDeployment(podKey, members, credentialsRevision)const name = deployment.metadata!.name!- try {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({- namespace: config.llmHooksNamespace,- body: deployment,- })- )- console.log(`${LOG} Created Deployment "${name}"`)- return- } catch (error) {- if (getErrorCode(error) !== 409) throw error- }- await replaceWithConflictRetry({- description: `Deployment "${name}"`,- logPrefix: LOG,- body: deployment,- mergeExisting: preserveDeploymentAnnotations,- isUpToDate: deploymentMatchesDesired,+ await ensureResource({read: () =>observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({ name, namespace: config.llmHooksNamespace })+ this.appsApi.readNamespacedDeployment({+ name,+ namespace: config.llmHooksNamespace,+ })),- replace: body =>- this.appsApi.replaceNamespacedDeployment({- name,+ create: async () => {+ await observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({+ namespace: config.llmHooksNamespace,+ body: deployment,+ })+ )+ hccLogger.info('Deployment created', {+ scope: LOG,+ deployment: name,namespace: config.llmHooksNamespace,- body,+ })+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `Deployment "${name}"`,+ logPrefix: LOG,+ body: deployment,+ mergeExisting: preserveDeploymentAnnotations,+ isUpToDate: deploymentMatchesDesired,+ read,+ replace: body =>+ this.appsApi.replaceNamespacedDeployment({+ name,+ namespace: config.llmHooksNamespace,+ body,+ }),}),+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),})}private async ensureService(podKey: string, port: number): Promise<void> {const service = this.buildService(podKey, port)const name = service.metadata!.name!- try {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: config.llmHooksNamespace,- body: service,- })- )- console.log(`${LOG} Created Service "${name}"`)- return- } catch (error) {- if (getErrorCode(error) !== 409) throw error- }- await replaceWithConflictRetry({- description: `Service "${name}"`,- logPrefix: LOG,- body: service,- mergeExisting: preserveServiceAssignedFields,- isUpToDate: serviceMatchesDesired,+ await ensureResource({read: () =>observeExistenceRead('Service', () =>- this.coreApi.readNamespacedService({ name, namespace: config.llmHooksNamespace })+ this.coreApi.readNamespacedService({+ name,+ namespace: config.llmHooksNamespace,+ })),- replace: body =>- this.coreApi.replaceNamespacedService({ name, namespace: config.llmHooksNamespace, body }),+ create: async () => {+ await observeCreate('Service', () =>+ this.coreApi.createNamespacedService({+ namespace: config.llmHooksNamespace,+ body: service,+ })+ )+ hccLogger.info('Service created', {+ scope: LOG,+ service: name,+ namespace: config.llmHooksNamespace,+ })+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `Service "${name}"`,+ logPrefix: LOG,+ body: service,+ mergeExisting: preserveServiceAssignedFields,+ isUpToDate: serviceMatchesDesired,+ read,+ replace: body =>+ this.coreApi.replaceNamespacedService({+ name,+ namespace: config.llmHooksNamespace,+ body,+ }),+ }),+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),})}diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.tsindex 2361471ba..135108158 100644--- a/host-context-controller/src/metrics.ts+++ b/host-context-controller/src/metrics.ts@@ -315,7 +315,7 @@ export const CREATE_KINDS = [export type CreateKind = (typeof CREATE_KINDS)[number]export const createsTotal = counter({name: 'clerum_hcc_creates_total',- help: 'Completed Kubernetes create outcomes: created (resolved), conflict (409), or error; skipped is reserved and zero in this instrumentation stage, by kind.',+ help: 'Kubernetes create outcomes by kind: created (resolved), conflict (409), error, or skipped after successful presence-based suppression; errors and cancellations are not skipped.',labelNames: ['kind', 'outcome'] as const,})for (const kind of CREATE_KINDS) {diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 3a6277eaf..87796c7f1 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -14,6 +14,73 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './constimport { McpServerReconciler } from './reconciler'import { McpServerCRD } from './types'+describe.each(['Deployment', 'Service'] as const)('read-first %s contract', kind => {+ function fixture() {+ const appsApi = createMockAppsApi()+ const coreApi = createMockCoreApi()+ const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {+ appsApi: asAppsApi(appsApi),+ coreApi: asCoreApi(coreApi),+ customApi: asCustomApi(createMockCustomApi()),+ assumeInventoryAuthorityWhenUnconfigured: true,+ })+ const server = makeServer()+ const read =+ kind === 'Deployment' ? appsApi.readNamespacedDeployment : coreApi.readNamespacedService+ const create =+ kind === 'Deployment' ? appsApi.createNamespacedDeployment : coreApi.createNamespacedService+ const replace =+ kind === 'Deployment' ? appsApi.replaceNamespacedDeployment : coreApi.replaceNamespacedService+ const run = (isCurrent = () => true) =>+ kind === 'Deployment'+ ? (reconciler as any).ensureDeployment(server, '', isCurrent)+ : (reconciler as any).ensureService(server, isCurrent)+ return { read, create, replace, run, server }+ }++ it('propagates GET 403 without attempting a write', async () => {+ const f = fixture()+ const forbidden = { code: 403 }+ f.read.mockRejectedValue(forbidden)+ await expect(f.run()).rejects.toBe(forbidden)+ expect(f.read).toHaveBeenCalledExactlyOnceWith({+ name: f.server.name,+ namespace: f.server.namespace,+ })+ expect(f.create).not.toHaveBeenCalled()+ expect(f.replace).not.toHaveBeenCalled()+ })++ it('reads absence before its only create request', async () => {+ const f = fixture()+ const events: string[] = []+ f.read.mockImplementation(async () => {+ events.push('GET')+ throw { code: 404 }+ })+ f.create.mockImplementation(async () => {+ events.push('POST')+ return {}+ })+ await f.run()+ expect(events).toEqual(['GET', 'POST'])+ expect(f.replace).not.toHaveBeenCalled()+ })++ it('rechecks authority after an absent read', async () => {+ const f = fixture()+ let active = true+ f.read.mockImplementation(async () => {+ active = false+ throw { code: 404 }+ })+ await f.run(() => active)+ expect(f.read).toHaveBeenCalledTimes(1)+ expect(f.create).not.toHaveBeenCalled()+ expect(f.replace).not.toHaveBeenCalled()+ })+})+function deferred<T = void>(): {promise: Promise<T>resolve: (value: T | PromiseLike<T>) => void@@ -105,12 +172,18 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('should create Deployment when managed: true (no regression)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = makeServer({ name: 'mongo-mcp', managed: true })await reconciler.reconcile(server)expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()})it('rejects managed ownership changes after the first reconcile snapshot', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)const serverName = 'workflow-recipes'@@ -134,6 +207,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('patches the network-ready annotation with merge-patch semantics', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = {...makeServer({ name: 'stdio-mcp', managed: true }),annotations: { 'clerum.io/pre-deploy': 'true' },@@ -163,6 +239,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('Issue #408: stamps the observed generation alongside the network-ready ack', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = {...makeServer({ name: 'stdio-mcp', managed: true }),generation: 3,@@ -189,6 +268,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('Issue #408: re-acks when the stamped generation is stale (spec changed since last ack)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = {...makeServer({ name: 'stdio-mcp', managed: true }),generation: 3,@@ -246,6 +328,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('Issue #408: re-acks when the ack predates the stamp (upgrade path: stamp absent)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })// A pre-#408 HCC set network-ready:'true' without an observed-generation stamp.// The re-ack guard's stale-generation branch (typeof gen === 'number' && stamp !==// String(gen), i.e. undefined !== '3') must fire so the ack gains a generation.@@ -279,12 +364,18 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {})it('should create Deployment when managed: undefined (default: true)', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = makeServer({ name: 'mongo-mcp' })await reconciler.reconcile(server)expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()})it('completes a remote proxy reconcile without rewriting the desired image', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server = makeServer({name: 'remote-api',image: 'vendor/original-image:1',@@ -1148,6 +1239,9 @@ describe('restart-safe discovery status', () => {})it('prefers fresh in-memory status over persisted status from the same generation', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const server: McpServerCRD = {...makeServer({ name: 'live-status' }),generation: 3,@@ -1181,6 +1275,8 @@ describe('restart-safe discovery status', () => {['generation changes', 'stable-uid', 4],['the same name is recreated', 'replacement-uid', 3],])('does not reuse in-memory status when %s', async (_case, uid, generation) => {+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const original: McpServerCRD = {...makeServer({ name: 'identity-fenced' }),uid: 'original-uid',@@ -1525,6 +1621,9 @@ describe('full reconciliation inventory authority', () => {})it('retires a same-watch runtime revision superseded during Secret validation', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })let current: McpServerCRDconst original: McpServerCRD = {...makeServer({@@ -1959,6 +2058,9 @@ describe('unconfigured inventory authority fails closed (G1)', () => {})it('T-G1c: the explicit opt-in restores assume-current', async () => {+ // First materialization is absent; subsequent reads still use the live-state fixture.+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {appsApi: asAppsApi(appsApi),coreApi: asCoreApi(coreApi),diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex ea722336c..b5330f871 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -21,16 +21,18 @@ import {import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'import { config } from './config'import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'+import { hccLogger } from './logger'import {confirmAuthoritativeMcpServerAbsence,sameMcpServerDesiredRevision,} from './mcpServerSafety'-import { mcpserverMissingSecret } from './metrics'+import { createsTotal, mcpserverMissingSecret } from './metrics'import { McpServerCRD, McpServerStatus } from './types'import {canonicalStringify,configMapMatchesDesired,deploymentMatchesDesired,+ ensureResource,getErrorCode,observeCreate,observeExistenceRead,@@ -1483,49 +1485,46 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv): Promise<void> {const deployment = this.buildDeployment(server, credentialsRevision)- if (!isCurrent()) return- try {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({+ await ensureResource({+ mutationAllowed: isCurrent,+ read: () =>+ observeExistenceRead('Deployment', () =>+ this.appsApi.readNamespacedDeployment({+ name: server.name,+ namespace: server.namespace,+ })+ ),+ create: async () => {+ await observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({+ namespace: server.namespace,+ body: deployment,+ })+ )+ hccLogger.info('Deployment created', {+ scope: '[Reconciler]',+ deployment: server.name,namespace: server.namespace,- body: deployment,})- )- console.log(`[Reconciler] Created Deployment "${server.name}"`)- return- } catch (error: unknown) {- if (getErrorCode(error) !== 409) {- throw error- }- // Already exists — read+replace with conflict retry. Races with the- // K8s Deployment controller (which bumps resourceVersion as pods +- // status conditions evolve) and with concurrent reconciles for the- // same server (CRD watch + SecretInformer) routinely produce stale- // resourceVersion errors. Helper does jittered backoff up to its- // attempt cap then re-throws — silently swallowing would hide- // immutable-field changes, quota exhaustion, etc.- await replaceWithConflictRetry({- description: `Deployment "${server.name}"`,- logPrefix: '[Reconciler]',- body: deployment,- mergeExisting: preserveDeploymentAnnotations,- isUpToDate: deploymentMatchesDesired,- read: () =>- observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `Deployment "${server.name}"`,+ logPrefix: '[Reconciler]',+ body: deployment,+ mergeExisting: preserveDeploymentAnnotations,+ isUpToDate: deploymentMatchesDesired,+ read,+ replace: body =>+ this.appsApi.replaceNamespacedDeployment({name: server.name,namespace: server.namespace,- })- ),- replace: body =>- this.appsApi.replaceNamespacedDeployment({- name: server.name,- namespace: server.namespace,- body,- }),- mutationAllowed: isCurrent,- })- }+ body,+ }),+ mutationAllowed: isCurrent,+ }),+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),+ })}/**@@ -1537,29 +1536,30 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv): Promise<void> {const service = this.buildService(server)- if (!isCurrent()) return- try {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: server.namespace,- body: service,- })- )- } catch (error: unknown) {- if (getErrorCode(error) === 409) {- await replaceWithConflictRetry({+ await ensureResource({+ mutationAllowed: isCurrent,+ read: () =>+ observeExistenceRead('Service', () =>+ this.coreApi.readNamespacedService({+ name: server.name,+ namespace: server.namespace,+ })+ ),+ create: () =>+ observeCreate('Service', () =>+ this.coreApi.createNamespacedService({+ namespace: server.namespace,+ body: service,+ })+ ),+ converge: read =>+ replaceWithConflictRetry({description: `Service "${server.name}"`,logPrefix: '[Reconciler]',body: service,mergeExisting: preserveServiceAssignedFields,isUpToDate: serviceMatchesDesired,- read: () =>- observeExistenceRead('Service', () =>- this.coreApi.readNamespacedService({- name: server.name,- namespace: server.namespace,- })- ),+ read,replace: body =>this.coreApi.replaceNamespacedService({name: server.name,@@ -1567,11 +1567,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvbody,}),mutationAllowed: isCurrent,- })- } else {- throw error- }- }+ }),+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),+ })}/**diff --git a/host-context-controller/src/sharedFileSystemReconciler.ts b/host-context-controller/src/sharedFileSystemReconciler.tsindex cea3655bb..f34e8b7ee 100644--- a/host-context-controller/src/sharedFileSystemReconciler.ts+++ b/host-context-controller/src/sharedFileSystemReconciler.ts@@ -38,6 +38,8 @@ import {wfcInitJobNamePrefix,wfcServiceName,} from './k8s/sharedFileSystemFactory'+import { hccLogger } from './logger'+import { createsTotal } from './metrics'import type {SharedFileSystemCRD,SharedFileSystemCondition,@@ -47,6 +49,7 @@ import type {import {applyNetworkPolicy,deploymentMatchesDesired,+ ensureResource,getErrorCode,observeCreate,observeExistenceRead,@@ -346,44 +349,35 @@ export class SharedFileSystemReconciler {private async ensureDeployment(sfs: SharedFileSystemCRD): Promise<string> {const dep = buildDeployment(sfs, this.factoryConfig)const name = wfcDeploymentName(sfs)- try {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({- namespace: this.factoryConfig.hostNamespace,+ const namespace = this.factoryConfig.hostNamespace+ await ensureResource<k8s.V1Deployment>({+ read: () =>+ observeExistenceRead('Deployment', () =>+ this.appsApi.readNamespacedDeployment({ namespace, name })+ ),+ create: async () => {+ await observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({ namespace, body: dep })+ )+ hccLogger.info('SharedFileSystem Deployment created', { scope: LOG, name, namespace })+ },+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),+ converge: read =>+ replaceWithConflictRetry({+ description: `Deployment "${name}"`,+ logPrefix: LOG,body: dep,- })- )- console.log(`${LOG} Created Deployment "${name}"`)- } catch (err) {- if (getErrorCode(err) !== 409) {- throw err- }- await replaceWithConflictRetry({- description: `Deployment "${name}"`,- logPrefix: LOG,- body: dep,- read: () =>- observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({- namespace: this.factoryConfig.hostNamespace,- name,- })- ),- // The public auth-key reconciler may use `kubectl rollout restart` on- // this HCC-owned Deployment. Preserve the pod-template restart marker- // when the periodic SFS reconcile replaces the raw desired manifest;- // otherwise HCC would immediately create a second Recreate outage and- // defeat the reconciler's one-rollout contract.- mergeExisting: preserveDeploymentAnnotations,- isUpToDate: deploymentMatchesDesired,- replace: body =>- this.appsApi.replaceNamespacedDeployment({- namespace: this.factoryConfig.hostNamespace,- name,- body,- }),- })- }+ read,+ // The public auth-key reconciler may use `kubectl rollout restart` on+ // this HCC-owned Deployment. Preserve the pod-template restart marker+ // when the periodic SFS reconcile replaces the raw desired manifest;+ // otherwise HCC would immediately create a second Recreate outage and+ // defeat the reconciler's one-rollout contract.+ mergeExisting: preserveDeploymentAnnotations,+ isUpToDate: deploymentMatchesDesired,+ replace: body => this.appsApi.replaceNamespacedDeployment({ namespace, name, body }),+ }),+ })return name}@@ -392,22 +386,31 @@ export class SharedFileSystemReconciler {private async ensureService(sfs: SharedFileSystemCRD): Promise<string> {const svc = buildService(sfs, this.factoryConfig)const name = wfcServiceName(sfs)- try {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: this.factoryConfig.hostNamespace,- body: svc,- })- )- console.log(`${LOG} Created Service "${name}"`)- } catch (err) {- if (getErrorCode(err) !== 409) {- throw err- }- // Don't replace Service — clusterIP is immutable and the spec we build- // is otherwise stable. Log and continue.- console.log(`${LOG} Service "${name}" already exists`)- }+ const namespace = this.factoryConfig.hostNamespace+ await ensureResource<k8s.V1Service>({+ read: () =>+ observeExistenceRead('Service', () =>+ this.coreApi.readNamespacedService({ namespace, name })+ ),+ create: async () => {+ await observeCreate('Service', () =>+ this.coreApi.createNamespacedService({ namespace, body: svc })+ )+ hccLogger.info('SharedFileSystem Service created', { scope: LOG, name, namespace })+ },+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),+ converge: async read => {+ // Preserve the Service, including immutable clusterIP. A POST409 must+ // reread: disappearance is benign, but other read failures propagate.+ try {+ await read()+ } catch (err) {+ if (err == null || getErrorCode(err) !== 404) throw err+ return+ }+ hccLogger.info('SharedFileSystem Service already exists', { scope: LOG, name, namespace })+ },+ })return name}diff --git a/host-context-controller/test/__fixtures__/testMocks.ts b/host-context-controller/test/__fixtures__/testMocks.tsindex f858ede80..6c5ddcc0c 100644--- a/host-context-controller/test/__fixtures__/testMocks.ts+++ b/host-context-controller/test/__fixtures__/testMocks.ts@@ -131,11 +131,18 @@ function hccOwnedLabels(name = ''): Record<string, string> {export function createMockAppsApi(): MockAppsApi {return {createNamespacedDeployment: vi.fn().mockResolvedValue({}),- readNamespacedDeployment: vi.fn(({ name }: { name?: string } = {}) =>- Promise.resolve({- status: { readyReplicas: 1 },- metadata: { resourceVersion: '1', labels: hccOwnedLabels(name) },- } as MockK8sResource)+ readNamespacedDeployment: vi.fn(+ ({ name, namespace }: { name?: string; namespace?: string } = {}) =>+ Promise.resolve({+ status: { readyReplicas: 1 },+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: hccOwnedLabels(name),+ },+ } as MockK8sResource)),replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),patchNamespacedDeployment: vi.fn().mockResolvedValue({}),@@ -147,11 +154,18 @@ export function createMockAppsApi(): MockAppsApi {export function createMockCoreApi(): MockCoreApi {return {createNamespacedService: vi.fn().mockResolvedValue({}),- readNamespacedService: vi.fn(({ name }: { name?: string } = {}) =>- Promise.resolve({- metadata: { resourceVersion: '1', labels: hccOwnedLabels(name) },- spec: { clusterIP: '10.0.0.1' },- } as MockK8sResource)+ readNamespacedService: vi.fn(+ ({ name, namespace }: { name?: string; namespace?: string } = {}) =>+ Promise.resolve({+ metadata: {+ name,+ namespace,+ uid: `uid-${name}`,+ resourceVersion: '1',+ labels: hccOwnedLabels(name),+ },+ spec: { clusterIP: '10.0.0.1' },+ } as MockK8sResource)),listNamespacedService: vi.fn().mockResolvedValue({ items: [] }),replaceNamespacedService: vi.fn().mockResolvedValue({}),diff --git a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.tsindex 7f885adce..b0e5605c3 100644--- a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts+++ b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts@@ -129,11 +129,11 @@ function createReconciler() {return { reconciler, appsApi, coreApi, networkingApi, rbacApi }}-/** Extract the channel-reader Deployment body from a createNamespacedDeployment spy. */-function getChannelReaderCreateBody(+/** Extract the channel-reader Deployment body from the convergence PUT. */+function getChannelReaderConvergedBody(appsApi: ReturnType<typeof createMockAppsApi>): k8s.V1Deployment {- const call = appsApi.createNamespacedDeployment.mock.calls.find(([arg]) => {+ const call = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {const body = arg.body as k8s.V1Deploymentreturn body.metadata?.namespace === 'channels'})@@ -155,7 +155,7 @@ function getChannelReaderReplaceBody(// ── B1: synced cache + host has ≥1 CommunicationChannel ──────────────────────describe('reconcile() — B1 integration: channel-reader Deployment shape (synced cache, CC ≥ 1)', () => {- it('creates a channel-reader Deployment in the channels namespace with replicas=1', async () => {+ it('converges a channel-reader Deployment in the channels namespace with replicas=1', async () => {const { reconciler, appsApi } = createReconciler()reconciler.setCountCommunicationChannels(() => 1)@@ -164,12 +164,12 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synceawait reconciler.reconcile(makeHost())// Must have been called with a channels-ns Deployment- const channelsCalls = appsApi.createNamespacedDeployment.mock.calls.filter(+ const channelsCalls = appsApi.replaceNamespacedDeployment.mock.calls.filter(([arg]) => (arg.body as k8s.V1Deployment).metadata?.namespace === 'channels')expect(channelsCalls).toHaveLength(1)- const dep = getChannelReaderCreateBody(appsApi)+ const dep = getChannelReaderConvergedBody(appsApi)expect(dep.spec?.replicas).toBe(1)})@@ -183,7 +183,7 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synceawait reconciler.reconcile(makeHost())- const dep = getChannelReaderCreateBody(appsApi)+ const dep = getChannelReaderConvergedBody(appsApi)const env = dep.spec?.template?.spec?.containers?.[0]?.env ?? []const namespaceEnv = env.find((e: k8s.V1EnvVar) => e.name === 'CLERUM_NAMESPACE')expect(namespaceEnv).toBeDefined()@@ -199,7 +199,7 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synceawait reconciler.reconcile(makeHost())- const dep = getChannelReaderCreateBody(appsApi)+ const dep = getChannelReaderConvergedBody(appsApi)const envFrom = dep.spec?.template?.spec?.containers?.[0]?.envFrom ?? []expect(envFrom).toContainEqual({ configMapRef: { name: 'clerum-channel-reader-config' } })})@@ -231,7 +231,7 @@ describe('reconcile() — B2 integration: replica preservation when CC cache is// Return a "live replicas=1" response for the channel-reader read; for the host// deployment read, return HCC-owned labels so reconcile() continues normally.appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {+ ({ name, namespace }: { name: string; namespace: string }) => {if (name === 'channel-reader-alpha-host') {return Promise.resolve({metadata: {@@ -250,6 +250,9 @@ describe('reconcile() — B2 integration: replica preservation when CC cache isreturn Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',@@ -268,7 +271,12 @@ describe('reconcile() — B2 integration: replica preservation when CC cache isawait reconciler.reconcile(makeHost())// The replace call body must carry replicas=1, not 0- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()+ expect(+ appsApi.replaceNamespacedDeployment.mock.calls.filter(+ ([request]) => request.namespace === 'channels'+ )+ ).toHaveLength(1)+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()const replaceBody = getChannelReaderReplaceBody(appsApi)expect(replaceBody.spec?.replicas).toBe(1)})@@ -285,12 +293,15 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>// channel-reader Deployment create succeeds; subsequent read (for status)// returns readyReplicas=1.appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {+ ({ name, namespace }: { name: string; namespace: string }) => {if (name === 'channel-reader-alpha-host') {return Promise.resolve({status: { readyReplicas: 1 },spec: { replicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/host': 'alpha-host',@@ -303,6 +314,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>return Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',@@ -328,7 +342,7 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>reconciler.setIsCommunicationChannelCacheSynced(() => true)appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {+ ({ name, namespace }: { name: string; namespace: string }) => {if (name === 'channel-reader-alpha-host') {const err = new Error('Not Found') as Error & { code?: number }err.code = 404@@ -338,6 +352,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>return Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',@@ -364,12 +381,15 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>reconciler.setIsCommunicationChannelCacheSynced(() => true)appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {+ ({ name, namespace }: { name: string; namespace: string }) => {if (name === 'channel-reader-alpha-host') {return Promise.resolve({status: { readyReplicas: 0 },spec: { replicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/host': 'alpha-host',@@ -381,6 +401,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>return Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',@@ -409,13 +432,16 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>reconciler.setIsCommunicationChannelCacheSynced(() => true)appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {+ ({ name, namespace }: { name: string; namespace: string }) => {if (name === 'channel-reader-alpha-host') {// Deployment exists but scaled to 0 — no CCs.return Promise.resolve({status: { readyReplicas: 0 },spec: { replicas: 0 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/host': 'alpha-host',@@ -428,6 +454,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>return Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',@@ -455,18 +484,22 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>reconciler.setCountCommunicationChannels(() => 1)reconciler.setIsCommunicationChannelCacheSynced(() => true)+ let channelReads = 0appsApi.readNamespacedDeployment.mockImplementation(- ({ name }: { name: string; namespace: string }) => {- if (name === 'channel-reader-alpha-host') {+ ({ name, namespace }: { name: string; namespace: string }) => {+ if (name === 'channel-reader-alpha-host' && ++channelReads > 1) {return Promise.reject(new Error('api server unavailable'))}return Promise.resolve({status: { readyReplicas: 1 },metadata: {+ name,+ namespace,+ uid: `uid-${name}`,resourceVersion: '1',labels: {'clerum.io/managed-by': 'host-context-controller',- 'clerum.io/host': name,+ 'clerum.io/host': 'alpha-host',},},})@@ -474,6 +507,7 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>)await reconciler.reconcile(makeHost())+ expect(channelReads).toBe(2)const status = reconciler.getStatus('alpha-host')// mcp-host Deployment is still deployed+readydiff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.tsindex 7fceb76a3..a87a002e1 100644--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts@@ -418,7 +418,8 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {await (reconciler as any).ensureDeployment(host, [], 'revision-a')- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledOnce()expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({namespace: 'mcp-host',name: 'chatllm',@@ -628,8 +629,11 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {'CLERUM_STATELESS_LIFECYCLE')- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })- appsApi.readNamespacedDeployment.mockResolvedValueOnce(runningStateful)+ appsApi.replaceNamespacedDeployment.mockClear()+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ request.name === stateful.name ? Promise.resolve(runningStateful) : readDeployment(request)+ )const stateless = makeStatelessHost({ name: 'transition-host' })customApi.getNamespacedCustomObject.mockResolvedValue({@@ -640,9 +644,11 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {await reconciler.reconcile(stateless)- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()- const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]- .body as k8s.V1Deployment+ const hostReplacements = appsApi.replaceNamespacedDeployment.mock.calls.filter(+ ([request]) => request.name === stateful.name+ )+ expect(hostReplacements).toHaveLength(1)+ const replacement = hostReplacements[0][0].body as k8s.V1Deploymentexpect(replacement.spec?.replicas).toBe(1)expect(replacement.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')expect(@@ -672,6 +678,17 @@ describe('HostReconciler stateless lifecycle — env injection', () => {status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },})+ // Model a genuinely new principal Deployment, then expose its created state.+ let principalCreated = false+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request => {+ if (request.name === stateless.name && !principalCreated) return Promise.reject({ code: 404 })+ return readDeployment(request)+ })+ appsApi.createNamespacedDeployment.mockImplementation(async request => {+ if (request.body.metadata?.name === stateless.name) principalCreated = true+ return request.body+ })await reconciler.reconcile(stateless)const creates = appsApi.createNamespacedDeployment.mock.calls.filter(@@ -938,7 +955,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {it('propagates the principal Deployment mutation failure so lifecycle convergence can retry', async () => {const { reconciler, appsApi } = createReconciler()const failure = Object.assign(new Error('Deployment API unavailable'), { code: 503 })- appsApi.createNamespacedDeployment.mockImplementation(+ appsApi.replaceNamespacedDeployment.mockImplementation(async ({ body }: { body: k8s.V1Deployment }) => {if (body.metadata?.name === 'stateless-host') throw failurereturn {}@@ -948,12 +965,13 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {try {await expect(reconciler.reconcile(makeStatelessHost())).rejects.toBe(failure)+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()} finally {errorSpy.mockRestore()}})- it('rechecks channel policy after a Deployment create conflict before replace', async () => {+ it('rechecks channel policy when the Deployment existence read observes a late channel', async () => {let channelCount = 0let hostDeploymentReads = 0const { reconciler, appsApi, customApi } = createReconciler({diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.tsindex f0dd97d86..93d43ebda 100644--- a/host-context-controller/test/hostReconciler.test.ts+++ b/host-context-controller/test/hostReconciler.test.ts@@ -244,6 +244,15 @@ describe('HostReconciler', () => {if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })})+ const channelReader = appsApi.replaceNamespacedDeployment.mock.calls.find(+ ([request]) => request.name === 'channel-reader-alpha-host'+ )![0].body+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request =>+ request.name === 'channel-reader-alpha-host'+ ? Promise.resolve(channelReader)+ : readDeployment(request)+ )coreApi.replaceNamespacedSecret.mockClear()appsApi.replaceNamespacedDeployment.mockClear()@@ -280,7 +289,7 @@ describe('HostReconciler', () => {).toBe(true)})- it('creates deployment, service, and pvc for valid host', async () => {+ it('converges existing deployment and service and creates pvc for valid host', async () => {const { reconciler, appsApi, coreApi } = createReconciler()const host = makeHost()@@ -291,10 +300,12 @@ describe('HostReconciler', () => {name: 'host-secret',})expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)- // reconcile creates two Services: the host (mcp-host ns) and its+ // Reconcile updates two existing Services: the host (mcp-host ns) and its// channel-reader handoff Service (channels ns).- expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(2)- expect(coreApi.createNamespacedService.mock.calls).toContainEqual([+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)+ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(2)+ expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([expect.objectContaining({namespace: 'mcp-host',body: expect.objectContaining({@@ -302,7 +313,7 @@ describe('HostReconciler', () => {}),}),])- expect(coreApi.createNamespacedService.mock.calls).toContainEqual([+ expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([expect.objectContaining({namespace: 'channels',body: expect.objectContaining({@@ -310,13 +321,14 @@ describe('HostReconciler', () => {}),}),])- // reconcile creates two Deployments: the host (mcp-host ns) and its+ // Reconcile updates two existing Deployments: the host (mcp-host ns) and its// channel-reader (channels ns).- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(2)- expect(appsApi.createNamespacedDeployment.mock.calls).toContainEqual([+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)+ expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([expect.objectContaining({ namespace: 'mcp-host' }),])- expect(appsApi.createNamespacedDeployment.mock.calls).toContainEqual([+ expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([expect.objectContaining({ namespace: 'channels' }),])expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })@@ -564,7 +576,7 @@ describe('HostReconciler', () => {]))- const channelReaderCreate = appsApi.createNamespacedDeployment.mock.calls.find(([arg]) => {+ const channelReaderCreate = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {const body = arg.body as k8s.V1Deploymentreturn body.metadata?.name === 'channel-reader-alpha-host'})@@ -697,7 +709,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const container = call.body.spec.template.spec.containers[0]expect(container.image).toBe('clerum/mcp-host-desktop:latest')})@@ -707,7 +719,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ browser: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const container = call.body.spec.template.spec.containers[0]expect(container.image).toBe('clerum/mcp-host-desktop:latest')})@@ -717,7 +729,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const ports = call.body.spec.template.spec.containers[0].portsexpect(ports).toHaveLength(2)expect(ports[0]).toMatchObject({ name: 'http', containerPort: 8080 })@@ -729,7 +741,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = coreApi.createNamespacedService.mock.calls[0][0]+ const call = coreApi.replaceNamespacedService.mock.calls[0][0]const ports = call.body.spec.portsexpect(ports).toHaveLength(2)expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })@@ -741,7 +753,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true, browser: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const env = call.body.spec.template.spec.containers[0].envconst envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))expect(envMap.get('CLERUM_DESKTOP_X11')).toBe('true')@@ -756,7 +768,7 @@ describe('HostReconciler — desktop support', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const env = call.body.spec.template.spec.containers[0].envconst envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))@@ -770,7 +782,7 @@ describe('HostReconciler — desktop support', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const env = call.body.spec.template.spec.containers[0].envconst controlEnv = env.find((e: { name: string }) => e.name === 'MCP_HOST_WORKFLOW_CONTROL_TOKEN'@@ -787,7 +799,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const secCtx = call.body.spec.template.spec.securityContextexpect(secCtx.runAsNonRoot).toBe(false)expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })@@ -798,7 +810,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const startupProbe = call.body.spec.template.spec.containers[0].startupProbeexpect(startupProbe.initialDelaySeconds).toBe(30)expect(startupProbe.failureThreshold).toBe(120)@@ -809,7 +821,7 @@ describe('HostReconciler — desktop support', () => {const host = makeDesktopHost({ x11: true })await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const resources = call.body.spec.template.spec.containers[0].resourcesexpect(resources.requests.memory).toBe('256Mi')expect(resources.requests.cpu).toBe('250m')@@ -895,7 +907,7 @@ describe('HostReconciler — desktop support', () => {const host = makeHost()await reconciler.reconcile(host)- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const container = call.body.spec.template.spec.containers[0]expect(container.image).toBe('clerum/mcp-host:0.6.0')expect(container.resources.requests.memory).toBe('128Mi')@@ -914,7 +926,7 @@ describe('HostReconciler — desktop support', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const secCtx = call.body.spec.template.spec.securityContextexpect(secCtx.runAsNonRoot).toBe(true)// 1001 = the mcp-host image's baked-in `nodejs` user; matches the@@ -931,7 +943,7 @@ describe('HostReconciler — desktop support', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]const container = call.body.spec.template.spec.containers[0]expect(container.name).toBe('mcp-host')expect(container.securityContext).toMatchObject({@@ -953,7 +965,7 @@ describe('HostReconciler — desktop support', () => {const host = makeHost()await reconciler.reconcile(host)- const call = coreApi.createNamespacedService.mock.calls[0][0]+ const call = coreApi.replaceNamespacedService.mock.calls[0][0]const ports = call.body.spec.portsexpect(ports).toHaveLength(1)expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })@@ -1015,7 +1027,7 @@ describe('HostReconciler — per-Host RBAC scaffolding', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const deployment = appsApi.createNamespacedDeployment.mock.calls[0][0].body+ const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].bodyexpect(deployment.spec.template.spec.serviceAccountName).toBe('host-alpha-host-sa')})@@ -1023,7 +1035,7 @@ describe('HostReconciler — per-Host RBAC scaffolding', () => {const { reconciler, appsApi } = createReconciler()await reconciler.reconcile(makeHost())- const deployment = appsApi.createNamespacedDeployment.mock.calls[0][0].body+ const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].bodyconst env = deployment.spec.template.spec.containers[0].env as Array<{name: stringvalue?: string@@ -1292,6 +1304,7 @@ describe('reconcileChannelReaderDeployment', () => {it('creates Deployment when absent (createNamespacedDeployment succeeds)', async () => {const { reconciler, appsApi, coreApi } = createReconciler()appsApi.createNamespacedDeployment.mockResolvedValue({})+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))@@ -1304,7 +1317,7 @@ describe('reconcileChannelReaderDeployment', () => {})})- it('replaces Deployment on 409 conflict (drift)', async () => {+ it('replaces a present Deployment with drift without creating', async () => {const { reconciler, appsApi, coreApi } = createReconciler()appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })appsApi.readNamespacedDeployment.mockResolvedValue({@@ -1456,16 +1469,13 @@ describe('reconcileChannelReaderDeployment', () => {converged.metadata = { ...converged.metadata, resourceVersion: '43' }appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })- appsApi.readNamespacedDeployment- .mockResolvedValueOnce(stale)- .mockResolvedValueOnce(stale)- .mockResolvedValueOnce(converged)+ appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(converged)appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })await (reconciler as any).reconcileChannelReaderDeployment(host)- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(3)+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()})@@ -1485,10 +1495,7 @@ describe('reconcileChannelReaderDeployment', () => {}appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })- appsApi.readNamespacedDeployment- .mockResolvedValueOnce(stale)- .mockResolvedValueOnce(stale)- .mockResolvedValueOnce(foreign)+ appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(foreign)appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })@@ -1501,6 +1508,7 @@ describe('reconcileChannelReaderDeployment', () => {it('reads existing Secret revision and writes annotation on initial create', async () => {const { reconciler, appsApi, coreApi } = createReconciler()+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })appsApi.createNamespacedDeployment.mockResolvedValue({})coreApi.readNamespacedSecret.mockResolvedValue({data: { 'telegram-bot-token': Buffer.from('tok-1').toString('base64') },@@ -1557,7 +1565,7 @@ describe('reconcile / reconcileDelete with channel-reader', () => {.spyOn(reconciler as any, 'reconcileChannelReaderDeployment').mockResolvedValue(undefined)await reconciler.reconcile(makeHost({ name: 'a' }))- expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }))+ expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }), expect.any(Function))})it('reconcileDelete(name, ns) deletes channel-reader-<name> Deployment in channels ns', async () => {diff --git a/host-context-controller/test/hostReconciler.wake.test.ts b/host-context-controller/test/hostReconciler.wake.test.tsindex 4fc7ae2aa..4c8fcb062 100644--- a/host-context-controller/test/hostReconciler.wake.test.ts+++ b/host-context-controller/test/hostReconciler.wake.test.ts@@ -510,8 +510,18 @@ describe('HostReconciler wake fast-path — durability and restart behavior', ()it('Deployment recreated while suspended stays at replicas=0 when the wake is already handled', async () => {const { reconciler, appsApi } = createReconciler()- // The mock create succeeds (no 409), i.e. the Deployment does not exist- // yet — exactly the "recreated while suspended" case.+ // The principal Deployment is absent until this reconcile creates it.+ let principalCreated = false+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!+ appsApi.readNamespacedDeployment.mockImplementation(request => {+ if (request.name === 'stateless-host' && !principalCreated)+ return Promise.reject({ code: 404 })+ return readDeployment(request)+ })+ appsApi.createNamespacedDeployment.mockImplementation(async request => {+ if (request.body.metadata?.name === 'stateless-host') principalCreated = true+ return request.body+ })await reconciler.reconcile(makeWakeHost({ wakeRequested: '2', state: 'suspended', wakeHandledGeneration: 2 }))