Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

feat(hcc): converge existing host and filesystem workloads before create

Alfredo Lopezb085237905a3
diff --git a/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts b/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts
index 3862221e4..7ae14ca05 100644
--- a/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts
+++ b/control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts
@@ -1,8 +1,4 @@
-// control-ui/e2e/qa-recorder-shared-fs-folders.spec.ts
-//
-// Optional QA recorder journey (MUTATING). Requires QA_RECORDER_CONFIRM_MUTATIONS=1.
-// Provisions a SharedFileSystem, then creates and renames a folder in its file
-// browser. The SharedFileSystem is removed via Control API in finally.
+// UI owns setup. API calls are limited to cleanup and negative authorization.
import { expect, test } from '@playwright/test'
import {
CONTROL_API_URL,
@@ -16,99 +12,221 @@ import {
uniqueE2EName,
} from './qa-recorder-helpers'
-function sharedFsDeletePath(name: string): string {
- return `/api/v1/admin/shared-filesystems/${encodeURIComponent(name)}`
+const collectionPath = '/api/v1/admin/shared-filesystems'
+type Filesystem = {
+ metadata?: { name?: string; uid?: string }
+ spec?: { size?: string; accessModes?: string[]; retainOnDelete?: boolean }
+}
+function isApiPath(rawUrl: string, path: string): boolean {
+ const url = new URL(rawUrl)
+ return url.origin === new URL(CONTROL_UI_URL).origin && url.pathname === `/control-api${path}`
}
test.describe('optional QA recorder: Control UI SharedFileSystem folders', () => {
- test('SharedFileSystem — folder create + rename', async ({ page }, testInfo) => {
+ test('SharedFileSystem — visible creation, folder rename and persistence', async ({
+ page,
+ }, testInfo) => {
requireRecorderConfirm(
'QA_RECORDER_CONFIRM_MUTATIONS',
- 'This journey provisions a SharedFileSystem and creates/renames a folder in it.'
+ 'This journey creates a SharedFileSystem and renames a folder.'
)
assertAllowedTarget('CONTROL_UI_URL', CONTROL_UI_URL)
assertAllowedTarget('CONTROL_API_URL', CONTROL_API_URL)
-
- const credentials = adminCredentials()
- const sharedFsName = uniqueE2EName('qa-recorder-shared-fs')
- const folderName = 'qa-recorder-folder'
- const renamedFolderName = 'qa-recorder-folder-renamed'
-
+ const name = uniqueE2EName('qa-recorder-shared-fs')
+ const detailPath = `/agent-files/${encodeURIComponent(name)}`
+ const apiPath = `${collectionPath}/${encodeURIComponent(name)}`
+ const filesPath = `${apiPath}/proxy/v1/files`
+ const folder = 'qa-recorder-folder'
+ const renamed = 'qa-recorder-folder-renamed'
+ let createdUid: string | undefined
try {
- await loginThroughUi(page, credentials)
-
- await page.goto(`${CONTROL_UI_URL}/agent-files/new`)
- await expect(
- page.getByRole('heading', { name: 'Create SharedFileSystem', exact: true })
- ).toBeVisible({ timeout: 20_000 })
-
- await page.getByPlaceholder('team-mission').fill(sharedFsName)
- await page.getByLabel('Shared filesystem size').fill('5')
- await page.getByLabel('Shared filesystem size unit').selectOption({ value: 'Gi' })
- await page
- .locator('.cu-create-actions')
- .getByRole('button', { name: 'Continue', exact: true })
- .click()
-
- await expect(page.getByText('Access and retention', { exact: true })).toBeVisible({
- timeout: 20_000,
+ await test.step('sign in and navigate to Agent Files', async () => {
+ await loginThroughUi(page, adminCredentials())
+ const nav = page.getByRole('navigation', { name: 'Main sections' })
+ await expect(nav).toBeVisible()
+ await nav.getByRole('link', { name: 'Files', exact: true }).click()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)
+ await expect(page.getByRole('heading', { name: /^Agent Files/ })).toBeVisible()
+ await expect(page.getByRole('button', { name: 'New', exact: true })).toBeEnabled()
})
- await page.getByLabel('Access mode').selectOption({ value: 'ReadWriteOnce' })
- await page.getByLabel(/Keep storage after deletion/i).check()
- await page
- .locator('.cu-create-actions')
- .getByRole('button', { name: 'Continue', exact: true })
- .click()
-
- await expect(page.getByText('Initial folders', { exact: true })).toBeVisible({
- timeout: 20_000,
+ await test.step('complete each wizard step and verify retained values', async () => {
+ await page.getByRole('button', { name: 'New', exact: true }).click()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files/new`)
+ await expect(
+ page.getByRole('heading', {
+ name: 'Create SharedFileSystem',
+ exact: true,
+ })
+ ).toBeVisible()
+ await expect(page.getByRole('region', { name: 'Storage request' })).toBeVisible()
+ await page.getByLabel('Name', { exact: true }).fill(name)
+ await page.getByLabel('Shared filesystem size', { exact: true }).fill('5')
+ await page.getByLabel('Shared filesystem size unit').selectOption('Gi')
+ await page.getByRole('button', { name: 'Continue', exact: true }).click()
+ await expect(page.getByRole('region', { name: 'Access and retention' })).toBeVisible()
+ await page.getByLabel('Access mode').selectOption('ReadWriteOnce')
+ await page.getByLabel('Keep storage after deletion').uncheck()
+ await expect(page.getByLabel('Access mode')).toHaveValue('ReadWriteOnce')
+ await expect(page.getByLabel('Keep storage after deletion')).not.toBeChecked()
+ await page.getByRole('button', { name: 'Back', exact: true }).click()
+ await expect(page.getByRole('region', { name: 'Storage request' })).toBeVisible()
+ await expect(page.getByLabel('Name', { exact: true })).toHaveValue(name)
+ await expect(page.getByLabel('Shared filesystem size', { exact: true })).toHaveValue('5')
+ await expect(page.getByLabel('Shared filesystem size unit')).toHaveValue('Gi')
+ await page.getByRole('button', { name: 'Continue', exact: true }).click()
+ await expect(page.getByRole('region', { name: 'Access and retention' })).toBeVisible()
+ await expect(page.getByLabel('Keep storage after deletion')).not.toBeChecked()
+ await page.getByRole('button', { name: 'Continue', exact: true }).click()
+ await expect(page.getByRole('region', { name: 'Initial folders' })).toBeVisible()
+ await expect(page.getByLabel('Directory name')).toBeVisible()
+ await expect(page.getByText('No folders added.', { exact: true })).toBeVisible()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files/new`)
})
- await page
- .locator('.cu-create-actions')
- .getByRole('button', { name: 'Create', exact: true })
- .click()
-
- await expect(
- page.getByText(`Shared filesystem "${sharedFsName}" created.`, { exact: true })
- ).toBeVisible({ timeout: 20_000 })
-
- await page.goto(`${CONTROL_UI_URL}/agent-files/${encodeURIComponent(sharedFsName)}`)
- await expect(page.getByRole('heading', { name: sharedFsName, exact: true })).toBeVisible({
- timeout: 20_000,
+ await test.step('submit and observe the real transition to the list', async () => {
+ const creation = page.waitForResponse(
+ r =>
+ isApiPath(r.url(), collectionPath) &&
+ r.request().method() === 'POST' &&
+ r.request().postDataJSON()?.name === name
+ )
+ await page.getByRole('button', { name: 'Create', exact: true }).click()
+ const response = await creation
+ expect(response.status()).toBe(201)
+ const resource = (await response.json()) as Filesystem
+ expect(resource.metadata?.name).toBe(name)
+ expect(resource.metadata?.uid).toEqual(expect.any(String))
+ createdUid = resource.metadata!.uid
+ expect(createdUid).not.toBe('')
+ expect(resource.spec).toMatchObject({
+ size: '5Gi',
+ accessModes: ['ReadWriteOnce'],
+ retainOnDelete: false,
+ })
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)
+ await expect(page.getByRole('heading', { name: /^Agent Files/ })).toBeVisible()
+ const row = page.getByRole('link', {
+ name: `Open shared filesystem ${name}`,
+ exact: true,
+ })
+ await expect(row).toBeVisible()
+ await expect(row).toContainText('5Gi')
})
- await expect(page.getByRole('button', { name: 'New folder', exact: true })).toBeVisible({
- timeout: 20_000,
+ await test.step('refresh until Ready and open the recorded row', async () => {
+ const row = page.getByRole('link', {
+ name: `Open shared filesystem ${name}`,
+ exact: true,
+ })
+ // The list has no auto-poll: exercise its visible Refresh action.
+ await expect(async () => {
+ await page.getByRole('button', { name: 'Refresh agent files', exact: true }).click()
+ await expect(row.getByText('Ready', { exact: true })).toBeVisible({
+ timeout: 1000,
+ })
+ }).toPass({ timeout: 180_000, intervals: [1000, 2000, 5000] })
+ const listing = page.waitForResponse(
+ r => isApiPath(r.url(), filesPath) && r.request().method() === 'GET'
+ )
+ await row.click()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}${detailPath}`)
+ await expect(page.getByRole('heading', { name, exact: true })).toBeVisible()
+ expect((await listing).status()).toBe(200)
+ await expect(page.getByRole('button', { name: 'New folder', exact: true })).toBeEnabled()
})
-
- await page.getByRole('button', { name: 'New folder', exact: true }).click()
- const newFolderDialog = page.getByRole('dialog', { name: 'New folder', exact: true })
- await expect(newFolderDialog).toBeVisible({ timeout: 20_000 })
- await newFolderDialog.getByLabel('Folder name').fill(folderName)
- await newFolderDialog.getByRole('button', { name: 'Create', exact: true }).click()
-
- await expect(page.getByRole('button', { name: folderName, exact: true })).toBeVisible({
- timeout: 20_000,
+ await test.step('create and rename a folder with visible controls', async () => {
+ await page.getByRole('button', { name: 'New folder', exact: true }).click()
+ const dialog = page.getByRole('dialog', {
+ name: 'New folder',
+ exact: true,
+ })
+ await expect(dialog).toBeVisible()
+ await dialog.getByLabel('Folder name').fill(folder)
+ const mkdir = page.waitForResponse(
+ r => isApiPath(r.url(), `${filesPath}/mkdir`) && r.request().method() === 'POST'
+ )
+ await dialog.getByRole('button', { name: 'Create', exact: true }).click()
+ expect((await mkdir).ok()).toBe(true)
+ await expect(dialog).toBeHidden()
+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeVisible()
+ await page.getByRole('button', { name: `Actions for ${folder}`, exact: true }).click()
+ await page.getByRole('menuitem', { name: 'Rename', exact: true }).click()
+ const rename = page.getByRole('dialog', {
+ name: `Rename ${folder}`,
+ exact: true,
+ })
+ await expect(rename).toBeVisible()
+ await expect(rename.getByLabel('New name')).toHaveValue(folder)
+ await rename.getByLabel('New name').fill(renamed)
+ const move = page.waitForResponse(
+ r => isApiPath(r.url(), `${filesPath}/move`) && r.request().method() === 'POST'
+ )
+ await rename.getByRole('button', { name: 'Rename', exact: true }).click()
+ expect((await move).ok()).toBe(true)
+ await expect(rename).toBeHidden()
+ await expect(page.getByRole('button', { name: renamed, exact: true })).toBeVisible()
+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeHidden()
})
-
- await page.getByRole('button', { name: `Rename ${folderName}`, exact: true }).click()
- const renameDialog = page.getByRole('dialog', {
- name: `Rename ${folderName}`,
- exact: true,
+ await test.step('reopen through the list and verify persisted folder state', async () => {
+ await page.getByRole('button', { name: 'Back to Agent Files', exact: true }).click()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}/agent-files`)
+ const row = page.getByRole('link', {
+ name: `Open shared filesystem ${name}`,
+ exact: true,
+ })
+ await expect(row).toBeVisible()
+ const listing = page.waitForResponse(
+ r => isApiPath(r.url(), filesPath) && r.request().method() === 'GET'
+ )
+ await row.click()
+ await expect(page).toHaveURL(`${CONTROL_UI_URL}${detailPath}`)
+ await expect(page.getByRole('heading', { name, exact: true })).toBeVisible()
+ const response = await listing
+ expect(response.status()).toBe(200)
+ const files = (await response.json()) as {
+ data: { entries: Array<{ name: string; kind: string }> }
+ }
+ expect(files.data.entries).toContainEqual(
+ expect.objectContaining({ name: renamed, kind: 'directory' })
+ )
+ expect(files.data.entries.some(entry => entry.name === folder)).toBe(false)
+ await expect(page.getByRole('button', { name: renamed, exact: true })).toBeVisible()
+ await expect(page.getByRole('button', { name: folder, exact: true })).toBeHidden()
+ await screenshotAndLog(page, testInfo, 'control-ui-shared-fs-folders')
})
- await expect(renameDialog).toBeVisible({ timeout: 20_000 })
- await renameDialog.getByLabel('New name').fill(renamedFolderName)
- await renameDialog.getByRole('button', { name: 'Rename', exact: true }).click()
-
- await expect(page.getByRole('button', { name: renamedFolderName, exact: true })).toBeVisible({
- timeout: 20_000,
- })
- await expect(
- page.getByRole('button', { name: `Rename ${folderName}`, exact: true })
- ).toBeHidden({ timeout: 20_000 })
-
- await screenshotAndLog(page, testInfo, 'control-ui-shared-fs-folders')
} finally {
- await api(page.request, 'DELETE', sharedFsDeletePath(sharedFsName))
+ if (createdUid) {
+ // Cleanup only. DELETE has no atomic UID precondition; require an isolated lane.
+ const current = await api<Filesystem>(page.request, 'GET', apiPath)
+ if (current.status !== 404) {
+ expect(current.status).toBe(200)
+ expect(current.data.metadata?.name).toBe(name)
+ expect(current.data.metadata?.uid).toBe(createdUid)
+ expect(current.data.spec?.retainOnDelete).toBe(false)
+ expect((await api(page.request, 'DELETE', apiPath)).status).toBe(204)
+ await expect
+ .poll(async () => (await api(page.request, 'GET', apiPath)).status, { timeout: 60_000 })
+ .toBe(404)
+ }
+ }
}
})
+ test('unauthenticated guard rejects the creation route and protected API', async ({ page }) => {
+ assertAllowedTarget('CONTROL_UI_URL', CONTROL_UI_URL)
+ // Explicit negative guard: this fresh test context has never signed in.
+ await page.goto(`${CONTROL_UI_URL}/agent-files/new`)
+ await expect(page).toHaveURL(
+ url => url.pathname === '/' && url.searchParams.get('next') === '/agent-files/new'
+ )
+ await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()
+ await expect(page.getByLabel('Username or email')).toBeVisible()
+ await expect(
+ page.getByRole('heading', {
+ name: 'Create SharedFileSystem',
+ exact: true,
+ })
+ ).toBeHidden()
+ // Negative authorization probe only; cannot advance the happy path.
+ const denied = await api(page.request, 'GET', collectionPath)
+ expect(denied.status).toBe(401)
+ expect(denied.data).not.toHaveProperty('items')
+ })
})
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index 41e577c2d..72a2f6ee1 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.564",
+ "version": "0.1.565",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.564",
+ "version": "0.1.565",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 48f6afc2f..82f1c7427 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.564",
+ "version": "0.1.565",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 626217c80..f6e93875e 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.305",
+ "version": "0.1.306",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.305",
+ "version": "0.1.306",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index e3773afa1..d95e80a36 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.305",
+ "version": "0.1.306",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
index 9c51b6d4b..9c11b2753 100644
--- a/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
+++ b/host-context-controller/src/__tests__/hostReconciler.mounts.test.ts
@@ -1260,7 +1260,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
coreApi: {
readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(),
createNamespacedServiceAccount: vi.fn(async () => ({})),
- createNamespacedService: vi.fn(async () => ({})),
+ createNamespacedService: vi.fn(),
+ readNamespacedService: vi.fn(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/host': 'team-mission',
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ spec: { clusterIP: '10.0.0.1' },
+ })),
+ replaceNamespacedService: vi.fn(async ({ body }) => body),
createNamespacedSecret: vi.fn(async () => ({})),
replaceNamespacedSecret: vi.fn(async () => ({})),
deleteNamespacedPersistentVolumeClaim: vi.fn(),
@@ -1273,20 +1287,44 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
} as unknown as k8s.RbacAuthorizationV1Api,
networkingApi: {
createNamespacedNetworkPolicy: vi.fn(async () => ({})),
- readNamespacedNetworkPolicy: vi.fn(async () => ({})),
+ readNamespacedNetworkPolicy: vi.fn(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/host': 'team-mission',
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ })),
replaceNamespacedNetworkPolicy: vi.fn(async () => ({})),
} as unknown as k8s.NetworkingV1Api,
appsApi: {
- // reconcile() now creates two Deployments (per-Host channel-reader +
+ // reconcile() now converges two Deployments (per-Host channel-reader +
// host). Capture only the host body — the channel-reader Deployment
// does not carry CLERUM_CONTEXT_FILES_MOUNTS.
- createNamespacedDeployment: vi.fn(async (req: { body: k8s.V1Deployment }) => {
+ createNamespacedDeployment: vi.fn(),
+ replaceNamespacedDeployment: vi.fn(async (req: { body: k8s.V1Deployment }) => {
if (req.body.metadata?.name === 'team-mission') {
captured.body = req.body
}
return {}
}),
- readNamespacedDeployment: vi.fn(async () => ({ status: { readyReplicas: 1 } })),
+ readNamespacedDeployment: vi.fn(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/host': 'team-mission',
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ status: { readyReplicas: 1 },
+ })),
} as unknown as k8s.AppsV1Api,
})
await reconciler.reconcile(makeHost())
@@ -1319,7 +1357,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
coreApi: {
readNamespacedSecret: readSecretWithChannelReaderRuntimeAuthLabels(desktopHost),
createNamespacedServiceAccount: vi.fn(async () => ({})),
- createNamespacedService: vi.fn(async () => ({})),
+ createNamespacedService: vi.fn(),
+ readNamespacedService: vi.fn(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/host': 'team-mission',
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ spec: { clusterIP: '10.0.0.1' },
+ })),
+ replaceNamespacedService: vi.fn(async ({ body }) => body),
createNamespacedSecret: vi.fn(async () => ({})),
replaceNamespacedSecret: vi.fn(async () => ({})),
deleteNamespacedPersistentVolumeClaim: vi.fn(),
@@ -1332,8 +1384,21 @@ describe('HostReconciler.reconcile — uses resolveContextMounts', () => {
} as unknown as k8s.RbacAuthorizationV1Api,
networkingApi: networkingApi as unknown as k8s.NetworkingV1Api,
appsApi: {
- createNamespacedDeployment: vi.fn(async () => ({})),
- readNamespacedDeployment: vi.fn(async () => ({ status: { readyReplicas: 1 } })),
+ createNamespacedDeployment: vi.fn(),
+ replaceNamespacedDeployment: vi.fn(async ({ body }) => body),
+ readNamespacedDeployment: vi.fn(async ({ name, namespace }) => ({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: {
+ 'clerum.io/host': 'team-mission',
+ 'clerum.io/managed-by': 'host-context-controller',
+ },
+ },
+ status: { readyReplicas: 1 },
+ })),
} as unknown as k8s.AppsV1Api,
})
diff --git a/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts b/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts
index b72c4561f..af0982594 100644
--- a/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/hostServiceNoopGate.test.ts
@@ -47,10 +47,11 @@ describe('Host ensureService no-op gate', () => {
})
})
- it('CREATE-SVC-1: successful create never reads or replaces', async () => {
+ it('CREATE-SVC-1: absent Service is read once then created without replace', async () => {
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
await (reconciler as any).ensureService(host)
expect(coreApi.createNamespacedService).toHaveBeenCalledOnce()
- expect(coreApi.readNamespacedService).not.toHaveBeenCalled()
+ expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()
expect(coreApi.replaceNamespacedService).not.toHaveBeenCalled()
})
@@ -61,6 +62,8 @@ describe('Host ensureService no-op gate', () => {
const log = vi.spyOn(console, 'log')
try {
await (reconciler as any).ensureService(host)
+ expect(coreApi.readNamespacedService).toHaveBeenCalledOnce()
+ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
expect(coreApi.replaceNamespacedService).not.toHaveBeenCalled()
expect(updatedServiceLogs(log, 'Service "chatllm"')).toEqual([])
} finally {
diff --git a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
index ef478fc3a..6add63ca3 100644
--- a/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
+++ b/host-context-controller/src/__tests__/llmHooksEffectiveEgress.test.ts
@@ -201,6 +201,9 @@ describe('llm-hooks effective pod egress (N5)', () => {
// Positive control: without this, the two assertions above would also pass if
// effectiveEgress() were simply blind to egress rules.
it('does surface the egress a declaring hook is granted (scoped DNS + its target)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const dialer = makeHook('dialer', {
target: {
image: { ref: IMG, port: 8080, egressBindings: [{ cidr: '8.8.8.8/32', ports: [443] }] },
diff --git a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
index f4e15bf35..027b952dc 100644
--- a/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.credentialsRotation.test.ts
@@ -57,8 +57,11 @@ function stored(value: string): string {
}
function createdDeployment(appsApi: ReturnType<typeof createMockAppsApi>): k8s.V1Deployment {
- const call = appsApi.createNamespacedDeployment.mock.calls[0]
- return (call[0] as { body: k8s.V1Deployment }).body
+ const call =
+ appsApi.replaceNamespacedDeployment.mock.calls.at(-1) ??
+ appsApi.createNamespacedDeployment.mock.calls.at(-1)
+ expect(call, 'the reconcile must write a real desired Deployment').toBeDefined()
+ return (call![0] as { body: k8s.V1Deployment }).body
}
function credentialsRevisionOf(deployment: k8s.V1Deployment): string | undefined {
@@ -92,9 +95,17 @@ function deploymentReadyWrites(customApi: ReturnType<typeof createMockCustomApi>
}
/** A Deployment whose rollout has fully converged. */
-function convergedDeployment(name?: string) {
+function convergedDeployment(name = 'linear') {
return {
- metadata: { name, resourceVersion: '1', generation: 2, labels: {} },
+ metadata: {
+ name,
+ resourceVersion: '1',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -114,6 +125,23 @@ describe('connector credentials revision (issue #223)', () => {
beforeEach(() => {
vi.clearAllMocks()
+ let live: k8s.V1Deployment | undefined
+ // This suite creates a connector, then observes the same API object on rotation.
+ appsApi.readNamespacedDeployment.mockImplementation(async () => {
+ if (!live) throw { code: 404 }
+ return structuredClone(live)
+ })
+ const persist = async ({ body }: { body: k8s.V1Deployment }) => {
+ live = {
+ ...structuredClone(body),
+ metadata: { ...body.metadata, resourceVersion: '7', generation: 1 },
+ status: { observedGeneration: 1, replicas: 1, updatedReplicas: 1, readyReplicas: 1 },
+ }
+ return live
+ }
+ appsApi.createNamespacedDeployment.mockImplementation(persist)
+ appsApi.replaceNamespacedDeployment.mockImplementation(persist)
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
coreApi.readNamespacedSecret.mockResolvedValue({
metadata: { name: 'linear-credentials' },
data: { 'api-key': stored('old-key') },
@@ -140,7 +168,7 @@ describe('connector credentials revision (issue #223)', () => {
await reconciler.reconcile(makeServer())
const first = credentialsRevisionOf(createdDeployment(appsApi))
- vi.clearAllMocks()
+ appsApi.replaceNamespacedDeployment.mockClear()
coreApi.readNamespacedSecret.mockResolvedValue({
data: { 'webhook-secret': stored('k2'), 'api-key': stored('k1') },
})
@@ -156,7 +184,7 @@ describe('connector credentials revision (issue #223)', () => {
await reconciler.reconcile(makeServer())
const before = credentialsRevisionOf(createdDeployment(appsApi))
- vi.clearAllMocks()
+ appsApi.replaceNamespacedDeployment.mockClear()
coreApi.readNamespacedSecret.mockResolvedValue({
data: { 'api-key': stored('rotated-key') },
})
@@ -183,7 +211,15 @@ describe('connector credentials revision (issue #223)', () => {
Object.assign(new Error('already exists'), { code: 409 })
)
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { name: 'linear', resourceVersion: '7', generation: 1, labels: {} },
+ metadata: {
+ name: 'linear',
+ resourceVersion: '7',
+ generation: 1,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: {
replicas: 1,
template: {
@@ -219,6 +255,9 @@ describe('generation-aware rollout readiness (issue #223)', () => {
beforeEach(() => {
vi.clearAllMocks()
+ // Initial create reads are absent; preserve the queued rollout observations after it.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
@@ -230,7 +269,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {
it('is NOT ready while the controller has not observed the current generation', async () => {
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 3, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 3,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
// The old pod is still Ready — the trap the pre-#223 check fell into.
status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 1 },
@@ -243,7 +292,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {
it('is NOT ready while replicas are not yet updated', async () => {
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 2 },
status: { observedGeneration: 2, updatedReplicas: 1, readyReplicas: 2 },
})
@@ -255,7 +314,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {
it('is NOT ready while a replica is unavailable', async () => {
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -288,7 +357,17 @@ describe('generation-aware rollout readiness (issue #223)', () => {
// is exactly the e2e E2 scenario; without `totalReplicas === updatedReplicas`
// the poll reports the failed rotation as healthy.
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -321,6 +400,9 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
beforeEach(() => {
vi.clearAllMocks()
+ // Initial create reads are absent; preserve the queued rollout observations after it.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
vi.useFakeTimers()
coreApi.readNamespacedSecret.mockResolvedValue({ data: { 'api-key': stored('old-key') } })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
@@ -340,7 +422,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
// converged by the time the poll runs.
appsApi.readNamespacedDeployment
.mockResolvedValueOnce({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },
})
@@ -355,7 +447,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
it('converges from the immediate False to a later True, instead of staying False', async () => {
appsApi.readNamespacedDeployment
.mockResolvedValueOnce({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: { observedGeneration: 1, updatedReplicas: 0, readyReplicas: 1 },
})
@@ -375,7 +477,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
it('gives up loudly on the CRD, with the rollout numbers, when it never converges', async () => {
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -402,7 +514,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
// behind, so the rollout never converges (total 2 vs updated 1) — the
// exact E2 shape. Deployment generation never changes: same rollout.
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -437,7 +559,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
it('grants a NEW rotation (bumped Deployment generation) a fresh poll budget mid-window', async () => {
const notConverged = (generation: number) => ({
- metadata: { generation, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: generation,
@@ -478,7 +610,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
// Deployment watch — so the CRD would keep the OLD generation's
// RolloutIncomplete forever even though the NEW rollout converges.
const notConverged = (generation: number) => ({
- metadata: { generation, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: generation,
@@ -490,7 +632,17 @@ describe('the readiness poll must publish its verdict on the CRD (issue #223)',
},
})
const converged = (generation: number) => ({
- metadata: { generation, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: generation,
@@ -564,7 +716,17 @@ describe('poll-window fence survives a superseding reconcile (F1, watch-recovery
// A rollout mid-flight (total 2 vs updated 1) that never converges on its own.
const midRollout = {
- metadata: { generation: 2, labels: {} },
+ metadata: {
+ name: 'linear',
+ namespace: 'mcp-server',
+ resourceVersion: '1',
+ uid: 'linear-deployment',
+ generation: 2,
+ labels: {
+ 'clerum.io/managed-by': 'host-context-controller',
+ 'clerum.io/mcpserver': 'linear',
+ },
+ },
spec: { replicas: 1 },
status: {
observedGeneration: 2,
@@ -578,6 +740,9 @@ describe('poll-window fence survives a superseding reconcile (F1, watch-recovery
beforeEach(() => {
vi.clearAllMocks()
+ // Initial create reads are absent; preserve the queued rollout observations after it.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
vi.useFakeTimers()
authorityGeneration = 1
currentServer = makeServer()
diff --git a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
index fa30ec7a5..675ac83ad 100644
--- a/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.pod-hardening.test.ts
@@ -88,6 +88,9 @@ describe('McpServer generated pod hardening', () => {
beforeEach(() => {
vi.clearAllMocks()
+ // These tests inspect the first generated workload, before it exists in the API.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
diff --git a/host-context-controller/src/__tests__/reconciler.remote.test.ts b/host-context-controller/src/__tests__/reconciler.remote.test.ts
index 044452efd..833e78037 100644
--- a/host-context-controller/src/__tests__/reconciler.remote.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.remote.test.ts
@@ -119,15 +119,15 @@ function mockHccOwnedRuntimeReads(
}
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1', labels },
+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },
status: { readyReplicas: 1 },
})
coreApi.readNamespacedConfigMap.mockResolvedValue({
- metadata: { resourceVersion: '1', labels },
+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },
data: {},
})
coreApi.readNamespacedService.mockResolvedValue({
- metadata: { resourceVersion: '1', labels },
+ metadata: { name, namespace: 'mcp-server', resourceVersion: '1', labels },
spec: { clusterIP: '10.0.0.1' },
})
}
@@ -719,6 +719,9 @@ describe('McpServerReconciler remote egress proxy', () => {
describe('reconcile call order for remote servers', () => {
it('should create ConfigMap before Deployment', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const callOrder: string[] = []
coreApi.createNamespacedConfigMap.mockImplementation(async () => {
@@ -745,6 +748,9 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should use the platform image without rewriting a divergent remote desired image', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const callOrder: string[] = []
const staleRemote = cloneServer(REMOTE_SERVER, {
image: LEGACY_REMOTE_EGRESS_IMAGE,
@@ -807,6 +813,9 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should also create Deployment and Service', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
await reconciler.reconcile(REMOTE_SERVER)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
@@ -836,6 +845,9 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should still create Deployment and Service', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
await reconciler.reconcile(LOCAL_SERVER)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
@@ -907,7 +919,12 @@ describe('McpServerReconciler remote egress proxy', () => {
it('should replace an existing remote egress proxy Deployment with the current image', async () => {
appsApi.createNamespacedDeployment.mockRejectedValue(make409Error())
appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: 'old-rv' },
+ metadata: {
+ name: REMOTE_SERVER.name,
+ namespace: REMOTE_SERVER.namespace,
+ resourceVersion: 'old-rv',
+ labels: { [MANAGED_BY_LABEL]: MANAGED_BY_VALUE, [MCPSERVER_LABEL]: REMOTE_SERVER.name },
+ },
spec: {
template: {
spec: {
@@ -1052,6 +1069,9 @@ describe('McpServerReconciler remote egress proxy', () => {
})
it('should clear status tracking after delete', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
let current: McpServerCRD | undefined = REMOTE_SERVER
reconciler.setResolveCurrentServer(() => current)
// First reconcile to populate status
@@ -1237,6 +1257,9 @@ describe('plugin image-host allowlist (2.3)', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const customApi = createMockCustomApi()
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const reconciler = buildReconciler(appsApi, coreApi, customApi)
const server = cloneServer(LOCAL_SERVER, { image: 'docker.io/evil/x:1' })
@@ -1277,6 +1300,9 @@ describe('plugin image-host allowlist (2.3)', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const customApi = createMockCustomApi()
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const reconciler = buildReconciler(appsApi, coreApi, customApi)
const server = cloneServer(LOCAL_SERVER, { image: 'registry.evenfire.ai/acme/x:1' })
@@ -1292,6 +1318,9 @@ describe('plugin image-host allowlist (2.3)', () => {
const appsApi = createMockAppsApi()
const coreApi = createMockCoreApi()
const customApi = createMockCustomApi()
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const reconciler = buildReconciler(appsApi, coreApi, customApi)
const cfg = (await import('../config')).config as { enforcePluginImageAllowlist: boolean }
cfg.enforcePluginImageAllowlist = true
diff --git a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
index a9dd07df6..fb12036ce 100644
--- a/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
+++ b/host-context-controller/src/__tests__/reconciler.sanitize.test.ts
@@ -90,6 +90,9 @@ describe('CRD Field Injection Prevention (sanitizeCrdSpec)', () => {
beforeEach(() => {
vi.clearAllMocks()
+ // These tests inspect the first generated workload, before it exists in the API.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
assumeInventoryAuthorityWhenUnconfigured: true,
appsApi: asAppsApi(appsApi),
diff --git a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
index c3f093563..6c82c70fb 100644
--- a/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
+++ b/host-context-controller/src/__tests__/sharedFileSystemReconciler.test.ts
@@ -1,9 +1,12 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type * as k8s from '@kubernetes/client-node'
import {
type SharedFileSystemFactoryConfig,
WFC_MOUNT_PATH,
+ buildDeployment,
buildEgressNetworkPolicy,
buildIngressNetworkPolicy,
+ buildService,
pvcName,
wfcDeploymentName,
wfcEgressPolicyName,
@@ -11,6 +14,7 @@ import {
wfcInitJobNamePrefix,
wfcServiceName,
} from '../k8s/sharedFileSystemFactory'
+import { createsTotal, existenceReadsTotal } from '../metrics'
import { SharedFileSystemReconciler } from '../sharedFileSystemReconciler'
import type { SharedFileSystemCRD } from '../types'
@@ -52,6 +56,14 @@ function err(code: number, msg = 'k8s error'): Error & { code: number } {
const notFound = () => err(404, 'not found')
const alreadyExists = () => err(409, 'already exists')
+function liveDeployment(
+ sfs = makeSfs(),
+ status: k8s.V1DeploymentStatus = { readyReplicas: 1, availableReplicas: 1 }
+): k8s.V1Deployment {
+ const desired = buildDeployment(sfs, factoryConfig)
+ return { ...desired, metadata: { ...desired.metadata, resourceVersion: '1' }, status }
+}
+
function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {
// Status/cleanup tests use already-converged policies with real identity and
// ownership. Cold-start tests explicitly arrange absence for their own reads.
@@ -61,11 +73,15 @@ function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {
])
const appsApi = {
createNamespacedDeployment: vi.fn().mockResolvedValue({}),
- // Truthful status (#592) reads readyReplicas; conflict-retry reads metadata.
- readNamespacedDeployment: vi.fn().mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 1, availableReplicas: 1 },
- }),
+ // Existence and readiness share an actual named Deployment; status tests
+ // vary its status without erasing its desired spec or ownership metadata.
+ readNamespacedDeployment: vi.fn(
+ async ({ name }: { name: string }): Promise<k8s.V1Deployment> => {
+ const sfs = systems.find(item => wfcDeploymentName(item) === name)
+ if (!sfs) throw new Error(`Missing Deployment fixture for ${name}`)
+ return liveDeployment(sfs)
+ }
+ ),
replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),
deleteNamespacedDeployment: vi.fn().mockResolvedValue({}),
}
@@ -73,6 +89,17 @@ function makeMocks(systems: SharedFileSystemCRD[] = [makeSfs()]) {
createNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
deleteNamespacedPersistentVolumeClaim: vi.fn().mockResolvedValue({}),
createNamespacedService: vi.fn().mockResolvedValue({}),
+ readNamespacedService: vi.fn(async ({ name }: { name: string }): Promise<k8s.V1Service> => {
+ const sfs = systems.find(item => wfcServiceName(item) === name)
+ if (!sfs) throw new Error(`Missing Service fixture for ${name}`)
+ const desired = buildService(sfs, factoryConfig)
+ return {
+ ...desired,
+ metadata: { ...desired.metadata, resourceVersion: '1' },
+ spec: { ...desired.spec, clusterIP: '10.96.1.23' },
+ }
+ }),
+ replaceNamespacedService: vi.fn(),
deleteNamespacedService: vi.fn().mockResolvedValue({}),
// Truthful status (#592): assessReadiness reads the PVC bind state, and
// detectWfcWedge lists the wfc Pod. Defaults = happy path (PVC Bound, no
@@ -152,16 +179,16 @@ describe('SharedFileSystemReconciler — monotonic status on re-reconcile (#592
phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)
return {}
})
- mocks.coreApi.createNamespacedService.mockImplementation(async () => {
+ mocks.coreApi.readNamespacedService.mockImplementation(async () => {
phasesSeenMidReconcile.push(reconciler.getStatus(sfs).phase)
- return {}
+ return buildService(sfs, factoryConfig)
})
// Re-reconcile the already-Ready SFS (mirrors a Context-change re-reconcile).
await reconciler.reconcile(sfs)
expect(reconciler.getStatus(sfs).phase).toBe('Ready')
- expect(phasesSeenMidReconcile.length).toBeGreaterThan(0)
+ expect(phasesSeenMidReconcile).toHaveLength(2)
for (const phase of phasesSeenMidReconcile) {
expect(phase).toBe('Ready')
}
@@ -179,6 +206,8 @@ describe('SharedFileSystemReconciler — happy path', () => {
it('creates PVC, Service, Deployment, and both NetworkPolicies — and NEVER an init Job', async () => {
const sfs = makeSfs()
+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
+ mocks.coreApi.readNamespacedService.mockRejectedValueOnce(notFound())
mocks.networkingApi.readNamespacedNetworkPolicy
.mockRejectedValueOnce(notFound())
.mockRejectedValueOnce(notFound())
@@ -201,6 +230,7 @@ describe('SharedFileSystemReconciler — happy path', () => {
it('builds the Deployment with a root initContainer that seeds the PVC in the same pod', async () => {
const sfs = makeSfs()
+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
await reconciler.reconcile(sfs)
const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body
const podSpec = dep.spec.template.spec
@@ -271,9 +301,13 @@ describe('SharedFileSystemReconciler — legacy init-Job cleanup', () => {
it('is non-fatal: a cleanup list failure does NOT block the Deployment rewrite', async () => {
const mocks = makeMocks()
mocks.batchApi.listNamespacedJob.mockRejectedValue(err(500, 'list boom'))
+ const drifted = liveDeployment()
+ drifted.spec!.replicas = 9
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce(drifted)
const reconciler = makeReconciler(mocks)
await expect(reconciler.reconcile(makeSfs())).resolves.toBeUndefined()
- expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(reconciler.getStatus(makeSfs()).phase).toBe('Ready')
})
@@ -335,17 +369,26 @@ describe('SharedFileSystemReconciler — idempotence', () => {
it('Deployment 409 (already exists) triggers a replace via the conflict-retry helper', async () => {
const mocks = makeMocks()
mocks.appsApi.createNamespacedDeployment.mockRejectedValueOnce(alreadyExists())
+ const drifted = liveDeployment()
+ drifted.spec!.replicas = 9
+ mocks.appsApi.readNamespacedDeployment
+ .mockRejectedValueOnce(notFound())
+ .mockResolvedValueOnce(drifted)
const reconciler = makeReconciler(mocks)
await reconciler.reconcile(makeSfs())
- expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalled()
+ expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(3)
+ expect(mocks.appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalled()
})
it('preserves external annotations without replacing the desired Deployment spec', async () => {
const mocks = makeMocks()
- mocks.appsApi.createNamespacedDeployment.mockRejectedValueOnce(alreadyExists())
mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '7', annotations: { 'operator.example/keep': 'yes' } },
+ metadata: {
+ ...liveDeployment().metadata,
+ resourceVersion: '7',
+ annotations: { 'operator.example/keep': 'yes' },
+ },
spec: {
replicas: 9,
strategy: { type: 'RollingUpdate' },
@@ -374,26 +417,27 @@ describe('SharedFileSystemReconciler — idempotence', () => {
await reconciler.reconcile(makeSfs())
- const desiredBody = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body
+ const desiredBody = buildDeployment(makeSfs(), factoryConfig)
+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
const replaceBody = mocks.appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
expect(replaceBody.metadata.annotations).toMatchObject({ 'operator.example/keep': 'yes' })
expect(replaceBody.spec.template.metadata.annotations).toEqual({
'kubectl.kubernetes.io/restartedAt': '2026-08-22T12:00:00Z',
'operator.example/pod-marker': 'keep',
})
- expect(replaceBody.spec.replicas).toBe(desiredBody.spec.replicas)
- expect(replaceBody.spec.strategy).toEqual(desiredBody.spec.strategy)
- expect(replaceBody.spec.selector).toEqual(desiredBody.spec.selector)
+ expect(replaceBody.spec.replicas).toBe(desiredBody.spec!.replicas)
+ expect(replaceBody.spec.strategy).toEqual(desiredBody.spec!.strategy)
+ expect(replaceBody.spec.selector).toEqual(desiredBody.spec!.selector)
expect(replaceBody.spec.template.metadata.labels).toEqual(
- desiredBody.spec.template.metadata.labels
+ desiredBody.spec!.template.metadata!.labels
)
expect(replaceBody.spec.template.spec.containers).toEqual(
- desiredBody.spec.template.spec.containers
+ desiredBody.spec!.template.spec!.containers
)
expect(replaceBody.spec.template.spec.initContainers).toEqual(
- desiredBody.spec.template.spec.initContainers
+ desiredBody.spec!.template.spec!.initContainers
)
- expect(replaceBody.spec.template.spec).toEqual(desiredBody.spec.template.spec)
+ expect(replaceBody.spec.template.spec).toEqual(desiredBody.spec!.template.spec)
})
it('never creates a Job across repeated reconciles', async () => {
@@ -529,6 +573,7 @@ describe('SharedFileSystemReconciler — fullReconcile', () => {
describe('SharedFileSystemReconciler — wfc env wiring (sanity)', () => {
it('passes WSF_MOUNT_PATH=/workspace and SFS identity into the Deployment env', async () => {
const mocks = makeMocks()
+ mocks.appsApi.readNamespacedDeployment.mockRejectedValueOnce(notFound())
const reconciler = makeReconciler(mocks)
await reconciler.reconcile(makeSfs())
const dep = mocks.appsApi.createNamespacedDeployment.mock.calls[0][0].body
@@ -555,10 +600,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod not created yet
const sfs = makeSfs()
await reconciler.reconcile(sfs)
@@ -569,10 +613,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({
items: [
{
@@ -607,10 +650,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({
items: [
{
@@ -640,10 +682,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Bound' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({
items: [
{
@@ -672,10 +713,9 @@ describe('SharedFileSystemReconciler — truthful status (#592)', () => {
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment
+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
await reconciler.reconcile(sfs)
expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
// Second reconcile falls through to the default happy-path mocks.
@@ -748,10 +788,9 @@ describe('SharedFileSystemReconciler — status-write dirty check + lastTransiti
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValueOnce({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValueOnce({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment
+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
+ .mockResolvedValueOnce(liveDeployment(makeSfs(), { readyReplicas: 0 }))
await reconciler.reconcile(sfs)
expect(reconciler.getStatus(sfs).phase).toBe('Initializing')
const writesAfterFirst = mocks.customApi.patchNamespacedCustomObjectStatus.mock.calls.length
@@ -794,10 +833,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readine
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Bound', capacity: { storage: '5Gi' } },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] }) // pod restarting
const sfs = makeSfs()
await reconciler.reconcile(sfs)
@@ -809,10 +847,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readine
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Bound', capacity: { storage: '5Gi' } },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({
items: [
{
@@ -837,10 +874,9 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readine
mocks.coreApi.readNamespacedPersistentVolumeClaim.mockResolvedValue({
status: { phase: 'Pending' },
})
- mocks.appsApi.readNamespacedDeployment.mockResolvedValue({
- metadata: { resourceVersion: '1' },
- status: { readyReplicas: 0 },
- })
+ mocks.appsApi.readNamespacedDeployment.mockResolvedValue(
+ liveDeployment(makeSfs(), { readyReplicas: 0 })
+ )
mocks.coreApi.listNamespacedPod.mockResolvedValue({ items: [] })
const sfs = makeSfs()
await reconciler.reconcile(sfs)
@@ -859,3 +895,153 @@ describe('SharedFileSystemReconciler — mountability decoupled from wfc readine
expect(reconciler.isMountable(sfs)).toBe(false)
})
})
+
+// Exercise the actual resource adapters without readiness reads masking their
+// physical GET counts. Integration status and recovery remain covered above.
+describe.each(['Deployment', 'Service'] as const)('SFS %s read-first contract', kind => {
+ beforeEach(() => {
+ createsTotal.reset()
+ existenceReadsTotal.reset()
+ })
+ function setup() {
+ const mocks = makeMocks()
+ const reconciler = makeReconciler(mocks) as unknown as {
+ ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>
+ ensureService(sfs: SharedFileSystemCRD): Promise<string>
+ }
+ const sfs = makeSfs()
+ const name = kind === 'Deployment' ? wfcDeploymentName(sfs) : wfcServiceName(sfs)
+ const read =
+ kind === 'Deployment'
+ ? mocks.appsApi.readNamespacedDeployment
+ : mocks.coreApi.readNamespacedService
+ const create =
+ kind === 'Deployment'
+ ? mocks.appsApi.createNamespacedDeployment
+ : mocks.coreApi.createNamespacedService
+ const replace =
+ kind === 'Deployment'
+ ? mocks.appsApi.replaceNamespacedDeployment
+ : mocks.coreApi.replaceNamespacedService
+ const run = () =>
+ kind === 'Deployment' ? reconciler.ensureDeployment(sfs) : reconciler.ensureService(sfs)
+ return { mocks, sfs, name, read, create, replace, run }
+ }
+ async function count(metric: typeof createsTotal, outcome: string) {
+ return (
+ (await metric.get()).values.find(
+ row => row.labels.kind === kind && row.labels.outcome === outcome
+ )?.value ?? 0
+ )
+ }
+
+ it('returns the existing name with one GET and no POST or PUT', async () => {
+ const h = setup()
+ await expect(h.run()).resolves.toBe(h.name)
+ expect(h.read).toHaveBeenCalledExactlyOnceWith({ name: h.name, namespace: 'mcp-host' })
+ expect(h.create).not.toHaveBeenCalled()
+ expect(h.replace).not.toHaveBeenCalled()
+ expect(await count(createsTotal, 'skipped')).toBe(1)
+ expect(await count(existenceReadsTotal, 'found')).toBe(1)
+ })
+
+ it('creates only after GET404 and returns the created name', async () => {
+ const h = setup()
+ h.read.mockRejectedValueOnce(notFound())
+ await expect(h.run()).resolves.toBe(h.name)
+ expect(h.read).toHaveBeenCalledTimes(1)
+ expect(h.create).toHaveBeenCalledExactlyOnceWith({
+ namespace: 'mcp-host',
+ body:
+ kind === 'Deployment'
+ ? buildDeployment(h.sfs, factoryConfig)
+ : buildService(h.sfs, factoryConfig),
+ })
+ expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])
+ expect(h.replace).not.toHaveBeenCalled()
+ expect(await count(createsTotal, 'created')).toBe(1)
+ expect(await count(createsTotal, 'skipped')).toBe(0)
+ })
+
+ it('consumes a fresh GET after POST409 rather than reusing absence', async () => {
+ const h = setup()
+ h.read.mockRejectedValueOnce(notFound())
+ h.create.mockRejectedValueOnce(alreadyExists())
+ await expect(h.run()).resolves.toBe(h.name)
+ expect(h.read).toHaveBeenCalledTimes(2)
+ expect(h.create).toHaveBeenCalledTimes(1)
+ expect(h.read.mock.invocationCallOrder[0]).toBeLessThan(h.create.mock.invocationCallOrder[0])
+ expect(h.create.mock.invocationCallOrder[0]).toBeLessThan(h.read.mock.invocationCallOrder[1])
+ expect(h.replace).not.toHaveBeenCalled()
+ expect(await count(createsTotal, 'conflict')).toBe(1)
+ expect(await count(createsTotal, 'skipped')).toBe(0)
+ })
+
+ it.each([403, 500])('propagates GET%s without any writes or skip', async code => {
+ const h = setup()
+ const failure = err(code)
+ h.read.mockRejectedValueOnce(failure)
+ await expect(h.run()).rejects.toBe(failure)
+ expect(h.read).toHaveBeenCalledTimes(1)
+ expect(h.create).not.toHaveBeenCalled()
+ expect(h.replace).not.toHaveBeenCalled()
+ expect(await count(createsTotal, 'skipped')).toBe(0)
+ expect(await count(existenceReadsTotal, 'error')).toBe(1)
+ })
+
+ it('propagates fresh GET403 after POST409', async () => {
+ const h = setup()
+ const failure = err(403)
+ h.read.mockRejectedValueOnce(notFound()).mockRejectedValueOnce(failure)
+ h.create.mockRejectedValueOnce(alreadyExists())
+ await expect(h.run()).rejects.toBe(failure)
+ expect(h.read).toHaveBeenCalledTimes(2)
+ expect(h.create).toHaveBeenCalledTimes(1)
+ expect(h.replace).not.toHaveBeenCalled()
+ expect(await count(createsTotal, 'skipped')).toBe(0)
+ })
+})
+
+describe('SFS preserve and retry contracts', () => {
+ it('keeps an existing Service unchanged even when its fields differ from desired', async () => {
+ const mocks = makeMocks()
+ const sfs = makeSfs()
+ const service = buildService(sfs, factoryConfig)
+ service.spec = { ...service.spec, clusterIP: '10.96.1.45', ports: [{ port: 9090 }] }
+ mocks.coreApi.readNamespacedService.mockResolvedValueOnce(service)
+ const reconciler = makeReconciler(mocks) as unknown as {
+ ensureService(sfs: SharedFileSystemCRD): Promise<string>
+ }
+ await expect(reconciler.ensureService(sfs)).resolves.toBe(wfcServiceName(sfs))
+ expect(mocks.coreApi.readNamespacedService).toHaveBeenCalledTimes(1)
+ expect(mocks.coreApi.createNamespacedService).not.toHaveBeenCalled()
+ expect(mocks.coreApi.replaceNamespacedService).not.toHaveBeenCalled()
+ expect(service.spec.clusterIP).toBe('10.96.1.45')
+ expect(service.spec.ports).toEqual([{ port: 9090 }])
+ })
+
+ it('reuses the first Deployment snapshot once and rereads its version after PUT409', async () => {
+ const mocks = makeMocks()
+ const sfs = makeSfs()
+ let version = 0
+ mocks.appsApi.readNamespacedDeployment.mockImplementation(async () => {
+ const live = liveDeployment(sfs)
+ live.metadata!.resourceVersion = String(++version)
+ live.spec!.replicas = 9
+ return live
+ })
+ mocks.appsApi.replaceNamespacedDeployment.mockRejectedValueOnce(alreadyExists())
+ const reconciler = makeReconciler(mocks) as unknown as {
+ ensureDeployment(sfs: SharedFileSystemCRD): Promise<string>
+ }
+ await expect(reconciler.ensureDeployment(sfs)).resolves.toBe(wfcDeploymentName(sfs))
+ expect(mocks.appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(mocks.appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(mocks.appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(
+ mocks.appsApi.replaceNamespacedDeployment.mock.calls.map(
+ ([{ body }]) => body.metadata.resourceVersion
+ )
+ ).toEqual(['1', '2'])
+ })
+})
diff --git a/host-context-controller/src/existenceReadMetrics.test.ts b/host-context-controller/src/existenceReadMetrics.test.ts
index 8992d1c78..381ad4c23 100644
--- a/host-context-controller/src/existenceReadMetrics.test.ts
+++ b/host-context-controller/src/existenceReadMetrics.test.ts
@@ -182,10 +182,10 @@ function readProductionSources(): Record<string, string> {
function assertReadInventory(sources: Record<string, string>): void {
const expectedWrapped: Record<string, number> = {
'utils.ts': 1,
- 'hostReconciler.ts': 8,
+ 'hostReconciler.ts': 7,
'reconciler.ts': 3,
'llmHookReconciler.ts': 3,
- 'sharedFileSystemReconciler.ts': 1,
+ 'sharedFileSystemReconciler.ts': 2,
'k8s/gfsK8sApi.ts': 5,
'networkPolicyReconciler.ts': 2,
}
diff --git a/host-context-controller/src/hostReconciler.readFirst.test.ts b/host-context-controller/src/hostReconciler.readFirst.test.ts
new file mode 100644
index 000000000..e6f443794
--- /dev/null
+++ b/host-context-controller/src/hostReconciler.readFirst.test.ts
@@ -0,0 +1,239 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import type * as k8s from '@kubernetes/client-node'
+import {
+ asAppsApi,
+ asCoreApi,
+ asCustomApi,
+ asNetworkingApi,
+ asRbacApi,
+ createMockAppsApi,
+ createMockCoreApi,
+ createMockCustomApi,
+ createMockNetworkingApi,
+ createMockRbacApi,
+ makeStubKc,
+} from '../test/__fixtures__/testMocks'
+import { asApiserverService } from './__tests__/asApiserverService'
+import { HostReconciler } from './hostReconciler'
+import { CREATE_KINDS, createsTotal } from './metrics'
+import type { HostCRD } from './types'
+
+const host: HostCRD = {
+ name: 'read-first-host',
+ namespace: 'mcp-host',
+ uid: 'host-uid',
+ spec: { host: 'read-first-host', contextRef: 'context', secretRef: 'host-secret' },
+}
+
+function fixture() {
+ const core = createMockCoreApi()
+ const apps = createMockAppsApi()
+ const reconciler = new HostReconciler(makeStubKc(), {
+ coreApi: asCoreApi(core),
+ appsApi: asAppsApi(apps),
+ networkingApi: asNetworkingApi(createMockNetworkingApi()),
+ rbacApi: asRbacApi(createMockRbacApi()),
+ customApi: asCustomApi(createMockCustomApi()),
+ })
+ vi.spyOn(reconciler as any, 'computeChannelReaderRevisionForHost').mockResolvedValue('revision')
+ return { core, apps, reconciler }
+}
+
+async function count(kind: string, outcome: string) {
+ const sample = (await createsTotal.get()).values.find(
+ v => v.labels.kind === kind && v.labels.outcome === outcome
+ )
+ expect(sample).toBeDefined()
+ return sample!.value
+}
+
+const cases = [
+ ['Host Service', 'Service', 'ensureService'],
+ ['channel-reader Service', 'Service', 'reconcileChannelReaderService'],
+ ['Host Deployment', 'Deployment', 'ensureDeployment'],
+ ['channel-reader Deployment', 'Deployment', 'reconcileChannelReaderDeployment'],
+] as const
+
+function invoke(reconciler: HostReconciler, method: string, revalidate?: () => void) {
+ return method === 'ensureDeployment'
+ ? (reconciler as any)[method](host, [], 'runtime-revision', undefined, undefined, revalidate)
+ : (reconciler as any)[method](host, revalidate)
+}
+
+describe('Host read-first Service and Deployment contracts', () => {
+ beforeEach(() => {
+ createsTotal.reset()
+ for (const kind of CREATE_KINDS)
+ for (const outcome of ['created', 'conflict', 'error', 'skipped'])
+ createsTotal.inc({ kind, outcome }, 0)
+ })
+
+ it.each(cases)(
+ '%s propagates initial GET403 without a mutation or skip',
+ async (_label, kind, method) => {
+ const { core, apps, reconciler } = fixture()
+ const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment
+ const create =
+ kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment
+ const replace =
+ kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment
+ const denied = { code: 403 }
+ read.mockRejectedValueOnce(denied)
+ await expect(invoke(reconciler, method)).rejects.toBe(denied)
+ expect(read).toHaveBeenCalledOnce()
+ expect(create).not.toHaveBeenCalled()
+ expect(replace).not.toHaveBeenCalled()
+ expect(await count(kind, 'skipped')).toBe(0)
+ expect(await count(kind, 'error')).toBe(0)
+ }
+ )
+
+ it.each(cases)(
+ '%s revalidates after an absent GET before any POST',
+ async (_label, kind, method) => {
+ const { core, apps, reconciler } = fixture()
+ const read = kind === 'Service' ? core.readNamespacedService : apps.readNamespacedDeployment
+ const create =
+ kind === 'Service' ? core.createNamespacedService : apps.createNamespacedDeployment
+ const replace =
+ kind === 'Service' ? core.replaceNamespacedService : apps.replaceNamespacedDeployment
+ let current = true
+ const superseded = new Error('admission retired during existence GET')
+ const revalidate = vi.fn(() => {
+ if (!current) throw superseded
+ })
+ read.mockImplementationOnce(async () => {
+ current = false
+ throw { code: 404 }
+ })
+ await expect(invoke(reconciler, method, revalidate)).rejects.toBe(superseded)
+ expect(read).toHaveBeenCalledOnce()
+ expect(revalidate).toHaveBeenCalledTimes(2)
+ expect(create).not.toHaveBeenCalled()
+ expect(replace).not.toHaveBeenCalled()
+ expect(await count(kind, 'skipped')).toBe(0)
+ }
+ )
+
+ it('rechecks Host admission after asynchronous Deployment body resolution', async () => {
+ const { apps, reconciler } = fixture()
+ let current = true
+ const superseded = new Error('admission retired during body resolution')
+ const revalidate = vi.fn(() => {
+ if (!current) throw superseded
+ })
+ apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ const resolveState = vi.fn(async () => {
+ expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()
+ current = false
+ return {
+ runtimeTokenRevision: 'fresh-revision',
+ lifecycle: { stateless: false, state: 'active' },
+ }
+ })
+ await expect(
+ (reconciler as any).ensureDeployment(
+ host,
+ [],
+ 'old-revision',
+ undefined,
+ resolveState,
+ revalidate
+ )
+ ).rejects.toBe(superseded)
+ expect(resolveState).toHaveBeenCalledOnce()
+ expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(apps.replaceNamespacedDeployment).not.toHaveBeenCalled()
+ expect(await count('Deployment', 'skipped')).toBe(0)
+ })
+
+ it('uses state resolved after GET404 in the first Deployment POST', async () => {
+ const { apps, reconciler } = fixture()
+ apps.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ const resolveState = vi.fn(async () => {
+ expect(apps.readNamespacedDeployment).toHaveBeenCalledOnce()
+ return {
+ runtimeTokenRevision: 'fresh-revision',
+ lifecycle: { stateless: false, state: 'active' },
+ }
+ })
+ await (reconciler as any).ensureDeployment(host, [], 'old-revision', undefined, resolveState)
+ expect(resolveState).toHaveBeenCalledOnce()
+ expect(apps.createNamespacedDeployment).toHaveBeenCalledOnce()
+ const body = apps.createNamespacedDeployment.mock.calls[0][0].body as k8s.V1Deployment
+ expect(body.spec?.template.metadata?.annotations?.['clerum.io/runtime-token-revision']).toBe(
+ 'fresh-revision'
+ )
+ expect(await count('Deployment', 'created')).toBe(1)
+ expect(await count('Deployment', 'skipped')).toBe(0)
+ })
+
+ it('retains a Host Service POST failure without recording presence suppression', async () => {
+ const { core, reconciler } = fixture()
+ core.readNamespacedService.mockRejectedValueOnce({ code: 404 })
+ core.createNamespacedService.mockRejectedValueOnce({ code: 503 })
+ await expect((reconciler as any).ensureService(host)).resolves.toBeUndefined()
+ expect(core.readNamespacedService).toHaveBeenCalledOnce()
+ expect(core.createNamespacedService).toHaveBeenCalledOnce()
+ expect(core.replaceNamespacedService).not.toHaveBeenCalled()
+ expect(await count('Service', 'error')).toBe(1)
+ expect(await count('Service', 'skipped')).toBe(0)
+ })
+
+ it('retains a Host Service convergence failure without recording a successful skip', async () => {
+ const { core, reconciler } = fixture()
+ core.replaceNamespacedService.mockRejectedValueOnce({ code: 503 })
+ await expect((reconciler as any).ensureService(host)).resolves.toBeUndefined()
+ expect(core.readNamespacedService).toHaveBeenCalledOnce()
+ expect(core.createNamespacedService).not.toHaveBeenCalled()
+ expect(core.replaceNamespacedService).toHaveBeenCalledOnce()
+ expect(await count('Service', 'error')).toBe(0)
+ expect(await count('Service', 'skipped')).toBe(0)
+ })
+
+ it('counts an equivalent present Host Service once without a POST or PUT', async () => {
+ const { core, reconciler } = fixture()
+ const desired = (reconciler as any).buildService(host) as k8s.V1Service
+ core.readNamespacedService.mockResolvedValue(asApiserverService(desired))
+ core.createNamespacedService.mockRejectedValue({ code: 409 })
+ await (reconciler as any).ensureService(host)
+ expect(core.readNamespacedService).toHaveBeenCalledOnce()
+ expect(core.createNamespacedService).not.toHaveBeenCalled()
+ expect(core.replaceNamespacedService).not.toHaveBeenCalled()
+ expect(await count('Service', 'skipped')).toBe(1)
+ })
+
+ it('rejects an invalid channel-reader retry response before another PUT', async () => {
+ const { apps, reconciler } = fixture()
+ const stale = (reconciler as any).buildChannelReaderDeployment(
+ host,
+ 'revision'
+ ) as k8s.V1Deployment
+ stale.metadata = { ...stale.metadata, resourceVersion: '1' }
+ stale.spec!.template.spec!.containers[0].image = 'channel-reader:stale'
+ apps.readNamespacedDeployment
+ .mockResolvedValueOnce(stale)
+ .mockResolvedValueOnce({ metadata: { resourceVersion: '2' } })
+ apps.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ await expect((reconciler as any).reconcileChannelReaderDeployment(host)).rejects.toThrow(
+ 'metadata.name'
+ )
+ expect(apps.readNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(apps.replaceNamespacedDeployment).toHaveBeenCalledOnce()
+ expect(apps.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(await count('Deployment', 'skipped')).toBe(0)
+ })
+
+ it('counts a Host Service create race as conflict and never as skipped', async () => {
+ const { core, reconciler } = fixture()
+ core.readNamespacedService.mockRejectedValueOnce({ code: 404 })
+ core.createNamespacedService.mockRejectedValueOnce({ code: 409 })
+ await (reconciler as any).ensureService(host)
+ expect(core.readNamespacedService).toHaveBeenCalledTimes(2)
+ expect(core.createNamespacedService).toHaveBeenCalledOnce()
+ expect(core.replaceNamespacedService).toHaveBeenCalledOnce()
+ expect(core.replaceNamespacedService.mock.calls[0][0].body.metadata.resourceVersion).toBe('1')
+ expect(await count('Service', 'conflict')).toBe(1)
+ expect(await count('Service', 'skipped')).toBe(0)
+ })
+})
diff --git a/host-context-controller/src/hostReconciler.test.ts b/host-context-controller/src/hostReconciler.test.ts
index e08462f3b..b185932a9 100644
--- a/host-context-controller/src/hostReconciler.test.ts
+++ b/host-context-controller/src/hostReconciler.test.ts
@@ -1501,19 +1501,27 @@ describe('HostReconciler Host inventory mutation authority', () => {
})
vi.spyOn(reconciler as any, 'hasChannelIngress').mockReturnValue(false)
appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
appsApi.readNamespacedDeployment.mockImplementationOnce(async () => {
current = {
...host,
generation: 4,
spec: { ...host.spec, contextRef: 'replacement-context' },
}
- return { metadata: { resourceVersion: 'rv-existing' } }
+ return {
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: 'deployment-uid',
+ resourceVersion: 'rv-existing',
+ },
+ }
})
await expect(reconciler.reconcile(host)).rejects.toThrow(/Host spec generation/)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).not.toHaveBeenCalled()
})
@@ -1550,7 +1558,14 @@ describe('HostReconciler Host inventory mutation authority', () => {
spec: { ...host.spec, contextRef: 'replacement-context' },
}
}
- return { metadata: { resourceVersion: `rv-existing-${deploymentReads}` } }
+ return {
+ metadata: {
+ name: host.name,
+ namespace: host.namespace,
+ uid: 'deployment-uid',
+ resourceVersion: `rv-existing-${deploymentReads}`,
+ },
+ }
})
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
const random = vi.spyOn(Math, 'random').mockReturnValue(0)
@@ -1561,7 +1576,7 @@ describe('HostReconciler Host inventory mutation authority', () => {
random.mockRestore()
}
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(1)
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
})
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 6fd28f36e..5f3f542a9 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -39,6 +39,7 @@ import {
import { hccLogger } from './logger'
import { issueMcpHostRuntimeTokens } from './mcpHostRuntimeTokenIssuerClient'
import {
+ createsTotal,
hostCleanupDeferredTotal,
hostDeleteCleanupTotal,
hostFleetBenignSupersessionsTotal,
@@ -77,6 +78,7 @@ import {
canonicalStringify,
canonicalizeValue,
deploymentMatchesDesired,
+ ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
@@ -2496,50 +2498,40 @@ export class HostReconciler {
}
}
- private async reconcileChannelReaderService(host: HostCRD): Promise<void> {
+ private async reconcileChannelReaderService(
+ host: HostCRD,
+ revalidate?: () => void
+ ): Promise<void> {
const name = `channel-reader-${host.name}`
const ns = config.channelsNamespace
const service = this.buildChannelReaderService(host)
-
- try {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: ns,
- body: service,
- })
- )
- console.log(`[HostReconciler] Created channel-reader Service "${name}"`)
- } catch (err) {
- if (getErrorCode(err) !== 409) {
- console.error(`[HostReconciler] Failed to create channel-reader Service "${name}":`, err)
- throw err
- }
- try {
- await replaceWithConflictRetry({
+ const mutationAllowed = () => {
+ revalidate?.()
+ return true
+ }
+ await ensureResource({
+ mutationAllowed,
+ read: () =>
+ observeExistenceRead('Service', () =>
+ this.coreApi.readNamespacedService({ name, namespace: ns })
+ ),
+ create: () =>
+ observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({ namespace: ns, body: service })
+ ),
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
+ converge: read =>
+ replaceWithConflictRetry({
description: `channel-reader Service "${name}"`,
logPrefix: '[HostReconciler]',
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
- read: () =>
- observeExistenceRead('Service', () =>
- this.coreApi.readNamespacedService({ name, namespace: ns })
- ),
- replace: body =>
- this.coreApi.replaceNamespacedService({
- name,
- namespace: ns,
- body,
- }),
- })
- } catch (replaceErr) {
- console.error(
- `[HostReconciler] Failed to update channel-reader Service "${name}":`,
- replaceErr
- )
- throw replaceErr
- }
- }
+ mutationAllowed,
+ read,
+ replace: body => this.coreApi.replaceNamespacedService({ name, namespace: ns, body }),
+ }),
+ })
}
/**
@@ -2550,73 +2542,62 @@ export class HostReconciler {
* clerum.io/credentials-revision pod template annotation, so the initial
* pod boots with the right hash and doesn't get rolled twice when
* SecretInformer fires later.
- * - On 409 from create, compares a fresh, server-normalized Deployment and
+ * - Reads before creation and compares a server-normalized Deployment; it
* replaces only meaningful drift. Conflict retries rebuild against the
* latest live replica count. A Deployment owned by a different host is
* never overwritten: the initial read skips, and a retry-time ownership
* change throws so write_skips_total does not count the refusal as a
* healthy no-op.
*/
- private async reconcileChannelReaderDeployment(host: HostCRD): Promise<void> {
+ private async reconcileChannelReaderDeployment(
+ host: HostCRD,
+ revalidate?: () => void
+ ): Promise<void> {
const name = `channel-reader-${host.name}`
const ns = config.channelsNamespace
-
const revision = await this.computeChannelReaderRevisionForHost(host.name)
const desired = this.buildChannelReaderDeployment(host, revision)
-
- try {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })
- )
- console.log(`[HostReconciler] Created channel-reader Deployment "${name}"`)
- } catch (err) {
- if (getErrorCode(err) !== 409) {
- console.error(`[HostReconciler] Failed to create channel-reader "${name}":`, err)
- throw err
- }
- let existing: k8s.V1Deployment
- try {
- existing = await observeExistenceRead('Deployment', () =>
+ const mutationAllowed = () => {
+ revalidate?.()
+ return true
+ }
+ await ensureResource({
+ mutationAllowed,
+ read: () =>
+ observeExistenceRead('Deployment', () =>
this.appsApi.readNamespacedDeployment({ name, namespace: ns })
- )
- } catch (readErr) {
- if (getErrorCode(readErr) === 404) {
- console.warn(
- `[HostReconciler] channel-reader "${name}" disappeared after create conflict; treating the reconcile as a transient failure`
- )
- throw readErr
+ ),
+ create: () =>
+ observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({ namespace: ns, body: desired })
+ ),
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
+ converge: async (read, observed) => {
+ // A post-create-conflict read still throws on 404. Later retry reads
+ // keep the existing benign-disappearance policy in the retry helper.
+ const existing = observed ?? (await read())
+ const ownerHost = existing.metadata?.labels?.[HOST_LABEL]
+ if (ownerHost && ownerHost !== host.name) {
+ log.warn('Channel-reader Deployment belongs to another Host; refusing update', { name })
+ return false
}
- console.error(`[HostReconciler] Failed to read channel-reader "${name}":`, readErr)
- throw readErr
- }
- const ownerHost = existing.metadata?.labels?.[HOST_LABEL]
- if (ownerHost && ownerHost !== host.name) {
- console.warn(
- `[HostReconciler] channel-reader "${name}" owned by host="${ownerHost}", skipping`
- )
- return
- }
- // The outer read above treats 404 as a transient throw so this Host
- // reconcile requeues and recreates. The helper read below is the #472
- // contract: 404 means gone, return, and the next Host reconcile
- // recreates. Same object, two 404 policies — accepted, not a bug.
- try {
+ // ensureResource already supplies a consume-once snapshot for the
+ // initial presence path. Only POST409 needs its owner-check read retained.
+ let snapshot: k8s.V1Deployment | undefined = observed === undefined ? existing : undefined
await replaceWithConflictRetry({
description: `channel-reader Deployment "${name}"`,
logPrefix: '[HostReconciler]',
- // B2: rebuild against each fresh read so an unsynced CC cache
- // preserves the live replica count even after a 409 retry.
body: this.buildChannelReaderDeployment(host, revision, existing.spec?.replicas),
- read: () =>
- observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({ name, namespace: ns })
- ),
- replace: body =>
- this.appsApi.replaceNamespacedDeployment({
- name,
- namespace: ns,
- body,
- }),
+ mutationAllowed,
+ read: () => {
+ if (snapshot !== undefined) {
+ const first = snapshot
+ snapshot = undefined
+ return Promise.resolve(first)
+ }
+ return read()
+ },
+ replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace: ns, body }),
mergeExisting: (_body, fresh) => {
const desiredWithLiveReplicas = this.buildChannelReaderDeployment(
host,
@@ -2631,9 +2612,6 @@ export class HostReconciler {
},
validateExisting: fresh => {
const freshOwnerHost = fresh.metadata?.labels?.[HOST_LABEL]
- // Never replace an object whose ownership changed during a retry.
- // Veto here — not via isUpToDate — so write_skips_total stays a
- // genuine no-op count.
if (freshOwnerHost && freshOwnerHost !== host.name) {
throw new Error(
`channel-reader Deployment ownership changed to host "${freshOwnerHost}"; refusing replace`
@@ -2642,11 +2620,8 @@ export class HostReconciler {
},
isUpToDate: deploymentMatchesDesired,
})
- } catch (replaceErr) {
- console.error(`[HostReconciler] Failed to update channel-reader "${name}":`, replaceErr)
- throw replaceErr
- }
- }
+ },
+ })
}
/**
@@ -3375,18 +3350,33 @@ export class HostReconciler {
}
}
- private async ensureService(host: HostCRD): Promise<void> {
+ private async ensureService(host: HostCRD, revalidate?: () => void): Promise<void> {
const service = this.buildService(host)
- try {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: host.namespace,
- body: service,
- })
- )
- console.log(`[HostReconciler] Created Service "${host.name}"`)
- } catch (error) {
- if (getErrorCode(error) === 409) {
+ const mutationAllowed = () => {
+ revalidate?.()
+ return true
+ }
+ // Initial read errors propagate. Only the existing POST/convergence paths
+ // retain their historical non-throwing failures.
+ await ensureResource({
+ mutationAllowed,
+ read: () =>
+ observeExistenceRead('Service', () =>
+ this.coreApi.readNamespacedService({ namespace: host.namespace, name: host.name })
+ ),
+ create: async () => {
+ try {
+ await observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({ namespace: host.namespace, body: service })
+ )
+ } catch (error) {
+ if (isBenignSupersessionError(error) || (error != null && getErrorCode(error) === 409))
+ throw error
+ log.error('Failed to create Host Service', { host: host.name, err: error })
+ }
+ },
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
+ converge: async read => {
try {
await replaceWithConflictRetry({
description: `Service "${host.name}"`,
@@ -3394,13 +3384,8 @@ export class HostReconciler {
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
- read: () =>
- observeExistenceRead('Service', () =>
- this.coreApi.readNamespacedService({
- namespace: host.namespace,
- name: host.name,
- })
- ),
+ mutationAllowed,
+ read,
replace: body =>
this.coreApi.replaceNamespacedService({
namespace: host.namespace,
@@ -3408,13 +3393,13 @@ export class HostReconciler {
body,
}),
})
- } catch (updateError) {
- console.error(`[HostReconciler] Failed to update Service "${host.name}":`, updateError)
+ } catch (error) {
+ if (isBenignSupersessionError(error)) throw error
+ log.error('Failed to update Host Service', { host: host.name, err: error })
+ return false
}
- } else {
- console.error(`[HostReconciler] Failed to create Service "${host.name}":`, error)
- }
- }
+ },
+ })
}
private async ensureDeployment(
@@ -3422,7 +3407,8 @@ export class HostReconciler {
mounts: ResolvedSfsMount[],
runtimeTokenRevision: string,
lifecycle?: EffectiveHostLifecycle,
- resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>
+ resolveStateBeforeMutation?: () => Promise<DeploymentMutationState>,
+ revalidate?: () => void
): Promise<void> {
const buildDesiredDeployment = async (): Promise<k8s.V1Deployment> => {
const state = resolveStateBeforeMutation ? await resolveStateBeforeMutation() : null
@@ -3433,47 +3419,43 @@ export class HostReconciler {
state?.lifecycle ?? lifecycle
)
}
- const deployment = await buildDesiredDeployment()
- try {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({
- namespace: host.namespace,
- body: deployment,
- })
- )
- console.log(`[HostReconciler] Created Deployment "${host.name}"`)
- } catch (error) {
- if (getErrorCode(error) !== 409) {
- console.error(`[HostReconciler] Failed to create Deployment "${host.name}":`, error)
- throw error
- }
- try {
- await replaceWithConflictRetry({
+ const mutationAllowed = () => {
+ revalidate?.()
+ return true
+ }
+ await ensureResource({
+ mutationAllowed,
+ read: () =>
+ observeExistenceRead('Deployment', () =>
+ this.appsApi.readNamespacedDeployment({ namespace: host.namespace, name: host.name })
+ ),
+ create: async () => {
+ // The initial GET may outlive the lifecycle or token-scope observation.
+ const deployment = await buildDesiredDeployment()
+ revalidate?.()
+ return observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({ namespace: host.namespace, body: deployment })
+ )
+ },
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
+ converge: read =>
+ replaceWithConflictRetry({
description: `Deployment "${host.name}"`,
logPrefix: '[HostReconciler]',
- body: deployment,
+ body: this.buildDeployment(host, mounts, runtimeTokenRevision, lifecycle),
resolveBody: buildDesiredDeployment,
mergeExisting: preserveHostDeploymentAnnotations,
isUpToDate: deploymentMatchesDesired,
- read: () =>
- observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({
- namespace: host.namespace,
- name: host.name,
- })
- ),
+ mutationAllowed,
+ read,
replace: body =>
this.appsApi.replaceNamespacedDeployment({
namespace: host.namespace,
name: host.name,
body,
}),
- })
- } catch (updateError) {
- console.error(`[HostReconciler] Failed to update Deployment "${host.name}":`, updateError)
- throw updateError
- }
- }
+ }),
+ })
}
private async deleteRuntimeResources(name: string, namespace: string): Promise<void> {
@@ -4426,7 +4408,7 @@ export class HostReconciler {
revalidateHostMutationBoundary()
await this.ensurePvc(host)
revalidateHostMutationBoundary()
- await this.ensureService(host)
+ await this.ensureService(host, revalidateHostMutationBoundary)
// NetworkPolicies before Deployments. Calico/Cilium evaluate egress and
// ingress against the policies that exist when the connection is opened,
@@ -4576,7 +4558,8 @@ export class HostReconciler {
mounts,
runtimeTokenProvision.revision,
lifecycle.effective,
- resolveDeploymentState
+ resolveDeploymentState,
+ revalidateHostMutationBoundary
)
revalidateHostMutationBoundary()
@@ -4619,8 +4602,8 @@ export class HostReconciler {
// expected error path — not just defense in depth.
const channelReaderFailures: unknown[] = []
for (const reconcileResource of [
- () => this.reconcileChannelReaderService(host),
- () => this.reconcileChannelReaderDeployment(host),
+ () => this.reconcileChannelReaderService(host, revalidateHostMutationBoundary),
+ () => this.reconcileChannelReaderDeployment(host, revalidateHostMutationBoundary),
]) {
try {
revalidateHostMutationBoundary()
diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.ts
index 7f9c673ee..f5e7c570a 100644
--- a/host-context-controller/src/llmHookReconciler.test.ts
+++ b/host-context-controller/src/llmHookReconciler.test.ts
@@ -248,6 +248,9 @@ describe('LlmHookReconciler', () => {
// ─── Dedup (§10) ───────────────────────────────────────────────────
it('co-locates two same-pod-key hooks on ONE Deployment/Service', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const a = makeHook({
name: 'a',
spec: {
@@ -275,6 +278,9 @@ describe('LlmHookReconciler', () => {
})
it('distinct pod keys → two Deployments', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const a = makeHook({
name: 'a',
spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },
@@ -286,6 +292,9 @@ describe('LlmHookReconciler', () => {
hooks.set('a', a)
hooks.set('b', b)
await reconciler.reconcile(a)
+ // The second pod key names a different absent workload.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
await reconciler.reconcile(b)
expect(new Set(deploymentNames(appsApi)).size).toBe(2)
})
@@ -293,6 +302,9 @@ describe('LlmHookReconciler', () => {
// ─── Path collision (§10) ──────────────────────────────────────────
it('fails the path-collision loser closed while serving the winner', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const a = makeHook({
name: 'a',
spec: {
@@ -427,6 +439,9 @@ describe('LlmHookReconciler', () => {
// ─── Concurrency (§10) ─────────────────────────────────────────────
it('serializes concurrent reconciles of two CRs sharing a pod key', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
let inFlight = 0
let maxInFlight = 0
appsApi.createNamespacedDeployment.mockImplementation(async () => {
@@ -477,9 +492,13 @@ describe('LlmHookReconciler', () => {
labels: { [MANAGED_BY_LABEL]: MANAGED_BY_VALUE, [HOOK_PODKEY_LABEL]: oldKey },
},
}
- appsApi.readNamespacedDeployment.mockResolvedValue(ownedOld)
- coreApi.readNamespacedService.mockResolvedValue(ownedOld)
- networkingApi.readNamespacedNetworkPolicy.mockResolvedValue(ownedOld)
+ const readOld = async ({ name }: { name: string }) => {
+ if (name !== podKeyResourceName(oldKey)) throw { code: 404 }
+ return { ...ownedOld, metadata: { ...ownedOld.metadata, name } }
+ }
+ appsApi.readNamespacedDeployment.mockImplementation(readOld)
+ coreApi.readNamespacedService.mockImplementation(readOld)
+ networkingApi.readNamespacedNetworkPolicy.mockImplementation(readOld)
await reconciler.reconcile(newHook, oldKey)
@@ -494,6 +513,9 @@ describe('LlmHookReconciler', () => {
// ─── NetworkPolicy ingress reverse-index (§10) ─────────────────────
it('NetworkPolicy ingress admits exactly the referencing Hosts', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// This scenario starts without a policy; the API reports absence explicitly.
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const a = makeHook({
@@ -519,6 +541,9 @@ describe('LlmHookReconciler', () => {
})
it('co-located members: NetworkPolicy label values are comma-free (member list is an annotation)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// This scenario starts without a policy; the API reports absence explicitly.
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
// Two hooks with the SAME image target share one pod key → one shared NP.
@@ -760,6 +785,9 @@ describe('LlmHookReconciler', () => {
})
it('hardens the hook pod: non-root, read-only root fs, dropped caps, no SA token (N8)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const a = makeHook({ name: 'a' })
hooks.set('a', a)
await reconciler.reconcile(a)
@@ -798,6 +826,9 @@ describe('LlmHookReconciler', () => {
})
it('grants scoped CoreDNS egress only when a hook declares egressBindings (N5)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// This scenario starts without a policy; the API reports absence explicitly.
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const withEgress = makeHook({
@@ -835,6 +866,9 @@ describe('LlmHookReconciler', () => {
// "grants NO egress (not even DNS)" is what let namespace-wide DNS through
// while the suite stayed green.
it('declares no Egress in the per-pod-key policy for a pure responder (no egressBindings)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// This scenario starts without a policy; the API reports absence explicitly.
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const responder = makeHook({ name: 'resp' }) // default image, no egressBindings
@@ -849,6 +883,9 @@ describe('LlmHookReconciler', () => {
})
it('allows a valid public-CIDR egress binding as an Egress NetworkPolicy rule', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// This scenario starts without a policy; the API reports absence explicitly.
networkingApi.readNamespacedNetworkPolicy.mockRejectedValue({ code: 404 })
const a = makeHook({
@@ -907,6 +944,9 @@ describe('LlmHookReconciler', () => {
})
it('writes conditions[] + observedDigest + readyReplicas for a ready image hook', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const digest = 'sha256:' + 'b'.repeat(64)
coreApi.listNamespacedPod.mockResolvedValue({
items: [
diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.ts
index 54e4ceb78..ecaa605de 100644
--- a/host-context-controller/src/llmHookReconciler.ts
+++ b/host-context-controller/src/llmHookReconciler.ts
@@ -750,7 +750,7 @@ export class LlmHookReconciler {
}
}
- // ─── Apply helpers (create-then-409 replaceWithConflictRetry) ─────────
+ // ─── Read-first resource convergence ───────────────────────────────
private async ensureDeployment(
podKey: string,
@@ -759,64 +759,86 @@ export class LlmHookReconciler {
): Promise<void> {
const deployment = this.buildDeployment(podKey, members, credentialsRevision)
const name = deployment.metadata!.name!
- try {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({
- namespace: config.llmHooksNamespace,
- body: deployment,
- })
- )
- console.log(`${LOG} Created Deployment "${name}"`)
- return
- } catch (error) {
- if (getErrorCode(error) !== 409) throw error
- }
- await replaceWithConflictRetry({
- description: `Deployment "${name}"`,
- logPrefix: LOG,
- body: deployment,
- mergeExisting: preserveDeploymentAnnotations,
- isUpToDate: deploymentMatchesDesired,
+ await ensureResource({
read: () =>
observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({ name, namespace: config.llmHooksNamespace })
+ this.appsApi.readNamespacedDeployment({
+ name,
+ namespace: config.llmHooksNamespace,
+ })
),
- replace: body =>
- this.appsApi.replaceNamespacedDeployment({
- name,
+ create: async () => {
+ await observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({
+ namespace: config.llmHooksNamespace,
+ body: deployment,
+ })
+ )
+ hccLogger.info('Deployment created', {
+ scope: LOG,
+ deployment: name,
namespace: config.llmHooksNamespace,
- body,
+ })
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Deployment "${name}"`,
+ logPrefix: LOG,
+ body: deployment,
+ mergeExisting: preserveDeploymentAnnotations,
+ isUpToDate: deploymentMatchesDesired,
+ read,
+ replace: body =>
+ this.appsApi.replaceNamespacedDeployment({
+ name,
+ namespace: config.llmHooksNamespace,
+ body,
+ }),
}),
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
})
}
private async ensureService(podKey: string, port: number): Promise<void> {
const service = this.buildService(podKey, port)
const name = service.metadata!.name!
- try {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: config.llmHooksNamespace,
- body: service,
- })
- )
- console.log(`${LOG} Created Service "${name}"`)
- return
- } catch (error) {
- if (getErrorCode(error) !== 409) throw error
- }
- await replaceWithConflictRetry({
- description: `Service "${name}"`,
- logPrefix: LOG,
- body: service,
- mergeExisting: preserveServiceAssignedFields,
- isUpToDate: serviceMatchesDesired,
+ await ensureResource({
read: () =>
observeExistenceRead('Service', () =>
- this.coreApi.readNamespacedService({ name, namespace: config.llmHooksNamespace })
+ this.coreApi.readNamespacedService({
+ name,
+ namespace: config.llmHooksNamespace,
+ })
),
- replace: body =>
- this.coreApi.replaceNamespacedService({ name, namespace: config.llmHooksNamespace, body }),
+ create: async () => {
+ await observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({
+ namespace: config.llmHooksNamespace,
+ body: service,
+ })
+ )
+ hccLogger.info('Service created', {
+ scope: LOG,
+ service: name,
+ namespace: config.llmHooksNamespace,
+ })
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Service "${name}"`,
+ logPrefix: LOG,
+ body: service,
+ mergeExisting: preserveServiceAssignedFields,
+ isUpToDate: serviceMatchesDesired,
+ read,
+ replace: body =>
+ this.coreApi.replaceNamespacedService({
+ name,
+ namespace: config.llmHooksNamespace,
+ body,
+ }),
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
})
}
diff --git a/host-context-controller/src/metrics.ts b/host-context-controller/src/metrics.ts
index 2361471ba..135108158 100644
--- a/host-context-controller/src/metrics.ts
+++ b/host-context-controller/src/metrics.ts
@@ -315,7 +315,7 @@ export const CREATE_KINDS = [
export type CreateKind = (typeof CREATE_KINDS)[number]
export const createsTotal = counter({
name: 'clerum_hcc_creates_total',
- help: 'Completed Kubernetes create outcomes: created (resolved), conflict (409), or error; skipped is reserved and zero in this instrumentation stage, by kind.',
+ help: 'Kubernetes create outcomes by kind: created (resolved), conflict (409), error, or skipped after successful presence-based suppression; errors and cancellations are not skipped.',
labelNames: ['kind', 'outcome'] as const,
})
for (const kind of CREATE_KINDS) {
diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.ts
index 3a6277eaf..87796c7f1 100644
--- a/host-context-controller/src/reconciler.test.ts
+++ b/host-context-controller/src/reconciler.test.ts
@@ -14,6 +14,73 @@ import { MANAGED_BY_LABEL, MCPSERVER_LABEL, WRC_MANAGED_BY_VALUE } from './const
import { McpServerReconciler } from './reconciler'
import { McpServerCRD } from './types'
+describe.each(['Deployment', 'Service'] as const)('read-first %s contract', kind => {
+ function fixture() {
+ const appsApi = createMockAppsApi()
+ const coreApi = createMockCoreApi()
+ const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
+ appsApi: asAppsApi(appsApi),
+ coreApi: asCoreApi(coreApi),
+ customApi: asCustomApi(createMockCustomApi()),
+ assumeInventoryAuthorityWhenUnconfigured: true,
+ })
+ const server = makeServer()
+ const read =
+ kind === 'Deployment' ? appsApi.readNamespacedDeployment : coreApi.readNamespacedService
+ const create =
+ kind === 'Deployment' ? appsApi.createNamespacedDeployment : coreApi.createNamespacedService
+ const replace =
+ kind === 'Deployment' ? appsApi.replaceNamespacedDeployment : coreApi.replaceNamespacedService
+ const run = (isCurrent = () => true) =>
+ kind === 'Deployment'
+ ? (reconciler as any).ensureDeployment(server, '', isCurrent)
+ : (reconciler as any).ensureService(server, isCurrent)
+ return { read, create, replace, run, server }
+ }
+
+ it('propagates GET 403 without attempting a write', async () => {
+ const f = fixture()
+ const forbidden = { code: 403 }
+ f.read.mockRejectedValue(forbidden)
+ await expect(f.run()).rejects.toBe(forbidden)
+ expect(f.read).toHaveBeenCalledExactlyOnceWith({
+ name: f.server.name,
+ namespace: f.server.namespace,
+ })
+ expect(f.create).not.toHaveBeenCalled()
+ expect(f.replace).not.toHaveBeenCalled()
+ })
+
+ it('reads absence before its only create request', async () => {
+ const f = fixture()
+ const events: string[] = []
+ f.read.mockImplementation(async () => {
+ events.push('GET')
+ throw { code: 404 }
+ })
+ f.create.mockImplementation(async () => {
+ events.push('POST')
+ return {}
+ })
+ await f.run()
+ expect(events).toEqual(['GET', 'POST'])
+ expect(f.replace).not.toHaveBeenCalled()
+ })
+
+ it('rechecks authority after an absent read', async () => {
+ const f = fixture()
+ let active = true
+ f.read.mockImplementation(async () => {
+ active = false
+ throw { code: 404 }
+ })
+ await f.run(() => active)
+ expect(f.read).toHaveBeenCalledTimes(1)
+ expect(f.create).not.toHaveBeenCalled()
+ expect(f.replace).not.toHaveBeenCalled()
+ })
+})
+
function deferred<T = void>(): {
promise: Promise<T>
resolve: (value: T | PromiseLike<T>) => void
@@ -105,12 +172,18 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('should create Deployment when managed: true (no regression)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = makeServer({ name: 'mongo-mcp', managed: true })
await reconciler.reconcile(server)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
})
it('rejects managed ownership changes after the first reconcile snapshot', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined)
const serverName = 'workflow-recipes'
@@ -134,6 +207,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('patches the network-ready annotation with merge-patch semantics', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = {
...makeServer({ name: 'stdio-mcp', managed: true }),
annotations: { 'clerum.io/pre-deploy': 'true' },
@@ -163,6 +239,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('Issue #408: stamps the observed generation alongside the network-ready ack', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = {
...makeServer({ name: 'stdio-mcp', managed: true }),
generation: 3,
@@ -189,6 +268,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('Issue #408: re-acks when the stamped generation is stale (spec changed since last ack)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = {
...makeServer({ name: 'stdio-mcp', managed: true }),
generation: 3,
@@ -246,6 +328,9 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('Issue #408: re-acks when the ack predates the stamp (upgrade path: stamp absent)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
// A pre-#408 HCC set network-ready:'true' without an observed-generation stamp.
// The re-ack guard's stale-generation branch (typeof gen === 'number' && stamp !==
// String(gen), i.e. undefined !== '3') must fire so the ack gains a generation.
@@ -279,12 +364,18 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {
})
it('should create Deployment when managed: undefined (default: true)', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = makeServer({ name: 'mongo-mcp' })
await reconciler.reconcile(server)
expect(appsApi.createNamespacedDeployment).toHaveBeenCalled()
})
it('completes a remote proxy reconcile without rewriting the desired image', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server = makeServer({
name: 'remote-api',
image: 'vendor/original-image:1',
@@ -1148,6 +1239,9 @@ describe('restart-safe discovery status', () => {
})
it('prefers fresh in-memory status over persisted status from the same generation', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const server: McpServerCRD = {
...makeServer({ name: 'live-status' }),
generation: 3,
@@ -1181,6 +1275,8 @@ describe('restart-safe discovery status', () => {
['generation changes', 'stable-uid', 4],
['the same name is recreated', 'replacement-uid', 3],
])('does not reuse in-memory status when %s', async (_case, uid, generation) => {
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const original: McpServerCRD = {
...makeServer({ name: 'identity-fenced' }),
uid: 'original-uid',
@@ -1525,6 +1621,9 @@ describe('full reconciliation inventory authority', () => {
})
it('retires a same-watch runtime revision superseded during Secret validation', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
let current: McpServerCRD
const original: McpServerCRD = {
...makeServer({
@@ -1959,6 +2058,9 @@ describe('unconfigured inventory authority fails closed (G1)', () => {
})
it('T-G1c: the explicit opt-in restores assume-current', async () => {
+ // First materialization is absent; subsequent reads still use the live-state fixture.
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
+ coreApi.readNamespacedService.mockRejectedValueOnce({ code: 404 })
const reconciler = new McpServerReconciler({} as k8s.KubeConfig, {
appsApi: asAppsApi(appsApi),
coreApi: asCoreApi(coreApi),
diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.ts
index ea722336c..b5330f871 100644
--- a/host-context-controller/src/reconciler.ts
+++ b/host-context-controller/src/reconciler.ts
@@ -21,16 +21,18 @@ import {
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
import { config } from './config'
import { MANAGED_BY_LABEL, MANAGED_BY_VALUE, MCPSERVER_LABEL } from './constants'
+import { hccLogger } from './logger'
import {
confirmAuthoritativeMcpServerAbsence,
sameMcpServerDesiredRevision,
} from './mcpServerSafety'
-import { mcpserverMissingSecret } from './metrics'
+import { createsTotal, mcpserverMissingSecret } from './metrics'
import { McpServerCRD, McpServerStatus } from './types'
import {
canonicalStringify,
configMapMatchesDesired,
deploymentMatchesDesired,
+ ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
@@ -1483,49 +1485,46 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
): Promise<void> {
const deployment = this.buildDeployment(server, credentialsRevision)
- if (!isCurrent()) return
- try {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({
+ await ensureResource({
+ mutationAllowed: isCurrent,
+ read: () =>
+ observeExistenceRead('Deployment', () =>
+ this.appsApi.readNamespacedDeployment({
+ name: server.name,
+ namespace: server.namespace,
+ })
+ ),
+ create: async () => {
+ await observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({
+ namespace: server.namespace,
+ body: deployment,
+ })
+ )
+ hccLogger.info('Deployment created', {
+ scope: '[Reconciler]',
+ deployment: server.name,
namespace: server.namespace,
- body: deployment,
})
- )
- console.log(`[Reconciler] Created Deployment "${server.name}"`)
- return
- } catch (error: unknown) {
- if (getErrorCode(error) !== 409) {
- throw error
- }
- // Already exists — read+replace with conflict retry. Races with the
- // K8s Deployment controller (which bumps resourceVersion as pods +
- // status conditions evolve) and with concurrent reconciles for the
- // same server (CRD watch + SecretInformer) routinely produce stale
- // resourceVersion errors. Helper does jittered backoff up to its
- // attempt cap then re-throws — silently swallowing would hide
- // immutable-field changes, quota exhaustion, etc.
- await replaceWithConflictRetry({
- description: `Deployment "${server.name}"`,
- logPrefix: '[Reconciler]',
- body: deployment,
- mergeExisting: preserveDeploymentAnnotations,
- isUpToDate: deploymentMatchesDesired,
- read: () =>
- observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Deployment "${server.name}"`,
+ logPrefix: '[Reconciler]',
+ body: deployment,
+ mergeExisting: preserveDeploymentAnnotations,
+ isUpToDate: deploymentMatchesDesired,
+ read,
+ replace: body =>
+ this.appsApi.replaceNamespacedDeployment({
name: server.name,
namespace: server.namespace,
- })
- ),
- replace: body =>
- this.appsApi.replaceNamespacedDeployment({
- name: server.name,
- namespace: server.namespace,
- body,
- }),
- mutationAllowed: isCurrent,
- })
- }
+ body,
+ }),
+ mutationAllowed: isCurrent,
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
+ })
}
/**
@@ -1537,29 +1536,30 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
): Promise<void> {
const service = this.buildService(server)
- if (!isCurrent()) return
- try {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: server.namespace,
- body: service,
- })
- )
- } catch (error: unknown) {
- if (getErrorCode(error) === 409) {
- await replaceWithConflictRetry({
+ await ensureResource({
+ mutationAllowed: isCurrent,
+ read: () =>
+ observeExistenceRead('Service', () =>
+ this.coreApi.readNamespacedService({
+ name: server.name,
+ namespace: server.namespace,
+ })
+ ),
+ create: () =>
+ observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({
+ namespace: server.namespace,
+ body: service,
+ })
+ ),
+ converge: read =>
+ replaceWithConflictRetry({
description: `Service "${server.name}"`,
logPrefix: '[Reconciler]',
body: service,
mergeExisting: preserveServiceAssignedFields,
isUpToDate: serviceMatchesDesired,
- read: () =>
- observeExistenceRead('Service', () =>
- this.coreApi.readNamespacedService({
- name: server.name,
- namespace: server.namespace,
- })
- ),
+ read,
replace: body =>
this.coreApi.replaceNamespacedService({
name: server.name,
@@ -1567,11 +1567,9 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv
body,
}),
mutationAllowed: isCurrent,
- })
- } else {
- throw error
- }
- }
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
+ })
}
/**
diff --git a/host-context-controller/src/sharedFileSystemReconciler.ts b/host-context-controller/src/sharedFileSystemReconciler.ts
index cea3655bb..f34e8b7ee 100644
--- a/host-context-controller/src/sharedFileSystemReconciler.ts
+++ b/host-context-controller/src/sharedFileSystemReconciler.ts
@@ -38,6 +38,8 @@ import {
wfcInitJobNamePrefix,
wfcServiceName,
} from './k8s/sharedFileSystemFactory'
+import { hccLogger } from './logger'
+import { createsTotal } from './metrics'
import type {
SharedFileSystemCRD,
SharedFileSystemCondition,
@@ -47,6 +49,7 @@ import type {
import {
applyNetworkPolicy,
deploymentMatchesDesired,
+ ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
@@ -346,44 +349,35 @@ export class SharedFileSystemReconciler {
private async ensureDeployment(sfs: SharedFileSystemCRD): Promise<string> {
const dep = buildDeployment(sfs, this.factoryConfig)
const name = wfcDeploymentName(sfs)
- try {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({
- namespace: this.factoryConfig.hostNamespace,
+ const namespace = this.factoryConfig.hostNamespace
+ await ensureResource<k8s.V1Deployment>({
+ read: () =>
+ observeExistenceRead('Deployment', () =>
+ this.appsApi.readNamespacedDeployment({ namespace, name })
+ ),
+ create: async () => {
+ await observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({ namespace, body: dep })
+ )
+ hccLogger.info('SharedFileSystem Deployment created', { scope: LOG, name, namespace })
+ },
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Deployment "${name}"`,
+ logPrefix: LOG,
body: dep,
- })
- )
- console.log(`${LOG} Created Deployment "${name}"`)
- } catch (err) {
- if (getErrorCode(err) !== 409) {
- throw err
- }
- await replaceWithConflictRetry({
- description: `Deployment "${name}"`,
- logPrefix: LOG,
- body: dep,
- read: () =>
- observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({
- namespace: this.factoryConfig.hostNamespace,
- name,
- })
- ),
- // The public auth-key reconciler may use `kubectl rollout restart` on
- // this HCC-owned Deployment. Preserve the pod-template restart marker
- // when the periodic SFS reconcile replaces the raw desired manifest;
- // otherwise HCC would immediately create a second Recreate outage and
- // defeat the reconciler's one-rollout contract.
- mergeExisting: preserveDeploymentAnnotations,
- isUpToDate: deploymentMatchesDesired,
- replace: body =>
- this.appsApi.replaceNamespacedDeployment({
- namespace: this.factoryConfig.hostNamespace,
- name,
- body,
- }),
- })
- }
+ read,
+ // The public auth-key reconciler may use `kubectl rollout restart` on
+ // this HCC-owned Deployment. Preserve the pod-template restart marker
+ // when the periodic SFS reconcile replaces the raw desired manifest;
+ // otherwise HCC would immediately create a second Recreate outage and
+ // defeat the reconciler's one-rollout contract.
+ mergeExisting: preserveDeploymentAnnotations,
+ isUpToDate: deploymentMatchesDesired,
+ replace: body => this.appsApi.replaceNamespacedDeployment({ namespace, name, body }),
+ }),
+ })
return name
}
@@ -392,22 +386,31 @@ export class SharedFileSystemReconciler {
private async ensureService(sfs: SharedFileSystemCRD): Promise<string> {
const svc = buildService(sfs, this.factoryConfig)
const name = wfcServiceName(sfs)
- try {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: this.factoryConfig.hostNamespace,
- body: svc,
- })
- )
- console.log(`${LOG} Created Service "${name}"`)
- } catch (err) {
- if (getErrorCode(err) !== 409) {
- throw err
- }
- // Don't replace Service — clusterIP is immutable and the spec we build
- // is otherwise stable. Log and continue.
- console.log(`${LOG} Service "${name}" already exists`)
- }
+ const namespace = this.factoryConfig.hostNamespace
+ await ensureResource<k8s.V1Service>({
+ read: () =>
+ observeExistenceRead('Service', () =>
+ this.coreApi.readNamespacedService({ namespace, name })
+ ),
+ create: async () => {
+ await observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({ namespace, body: svc })
+ )
+ hccLogger.info('SharedFileSystem Service created', { scope: LOG, name, namespace })
+ },
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
+ converge: async read => {
+ // Preserve the Service, including immutable clusterIP. A POST409 must
+ // reread: disappearance is benign, but other read failures propagate.
+ try {
+ await read()
+ } catch (err) {
+ if (err == null || getErrorCode(err) !== 404) throw err
+ return
+ }
+ hccLogger.info('SharedFileSystem Service already exists', { scope: LOG, name, namespace })
+ },
+ })
return name
}
diff --git a/host-context-controller/test/__fixtures__/testMocks.ts b/host-context-controller/test/__fixtures__/testMocks.ts
index f858ede80..6c5ddcc0c 100644
--- a/host-context-controller/test/__fixtures__/testMocks.ts
+++ b/host-context-controller/test/__fixtures__/testMocks.ts
@@ -131,11 +131,18 @@ function hccOwnedLabels(name = ''): Record<string, string> {
export function createMockAppsApi(): MockAppsApi {
return {
createNamespacedDeployment: vi.fn().mockResolvedValue({}),
- readNamespacedDeployment: vi.fn(({ name }: { name?: string } = {}) =>
- Promise.resolve({
- status: { readyReplicas: 1 },
- metadata: { resourceVersion: '1', labels: hccOwnedLabels(name) },
- } as MockK8sResource)
+ readNamespacedDeployment: vi.fn(
+ ({ name, namespace }: { name?: string; namespace?: string } = {}) =>
+ Promise.resolve({
+ status: { readyReplicas: 1 },
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: hccOwnedLabels(name),
+ },
+ } as MockK8sResource)
),
replaceNamespacedDeployment: vi.fn().mockResolvedValue({}),
patchNamespacedDeployment: vi.fn().mockResolvedValue({}),
@@ -147,11 +154,18 @@ export function createMockAppsApi(): MockAppsApi {
export function createMockCoreApi(): MockCoreApi {
return {
createNamespacedService: vi.fn().mockResolvedValue({}),
- readNamespacedService: vi.fn(({ name }: { name?: string } = {}) =>
- Promise.resolve({
- metadata: { resourceVersion: '1', labels: hccOwnedLabels(name) },
- spec: { clusterIP: '10.0.0.1' },
- } as MockK8sResource)
+ readNamespacedService: vi.fn(
+ ({ name, namespace }: { name?: string; namespace?: string } = {}) =>
+ Promise.resolve({
+ metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
+ resourceVersion: '1',
+ labels: hccOwnedLabels(name),
+ },
+ spec: { clusterIP: '10.0.0.1' },
+ } as MockK8sResource)
),
listNamespacedService: vi.fn().mockResolvedValue({ items: [] }),
replaceNamespacedService: vi.fn().mockResolvedValue({}),
diff --git a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
index 7f885adce..b0e5605c3 100644
--- a/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
+++ b/host-context-controller/test/hostReconciler.channelReaderIntegration.test.ts
@@ -129,11 +129,11 @@ function createReconciler() {
return { reconciler, appsApi, coreApi, networkingApi, rbacApi }
}
-/** Extract the channel-reader Deployment body from a createNamespacedDeployment spy. */
-function getChannelReaderCreateBody(
+/** Extract the channel-reader Deployment body from the convergence PUT. */
+function getChannelReaderConvergedBody(
appsApi: ReturnType<typeof createMockAppsApi>
): k8s.V1Deployment {
- const call = appsApi.createNamespacedDeployment.mock.calls.find(([arg]) => {
+ const call = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {
const body = arg.body as k8s.V1Deployment
return body.metadata?.namespace === 'channels'
})
@@ -155,7 +155,7 @@ function getChannelReaderReplaceBody(
// ── B1: synced cache + host has ≥1 CommunicationChannel ──────────────────────
describe('reconcile() — B1 integration: channel-reader Deployment shape (synced cache, CC ≥ 1)', () => {
- it('creates a channel-reader Deployment in the channels namespace with replicas=1', async () => {
+ it('converges a channel-reader Deployment in the channels namespace with replicas=1', async () => {
const { reconciler, appsApi } = createReconciler()
reconciler.setCountCommunicationChannels(() => 1)
@@ -164,12 +164,12 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synce
await reconciler.reconcile(makeHost())
// Must have been called with a channels-ns Deployment
- const channelsCalls = appsApi.createNamespacedDeployment.mock.calls.filter(
+ const channelsCalls = appsApi.replaceNamespacedDeployment.mock.calls.filter(
([arg]) => (arg.body as k8s.V1Deployment).metadata?.namespace === 'channels'
)
expect(channelsCalls).toHaveLength(1)
- const dep = getChannelReaderCreateBody(appsApi)
+ const dep = getChannelReaderConvergedBody(appsApi)
expect(dep.spec?.replicas).toBe(1)
})
@@ -183,7 +183,7 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synce
await reconciler.reconcile(makeHost())
- const dep = getChannelReaderCreateBody(appsApi)
+ const dep = getChannelReaderConvergedBody(appsApi)
const env = dep.spec?.template?.spec?.containers?.[0]?.env ?? []
const namespaceEnv = env.find((e: k8s.V1EnvVar) => e.name === 'CLERUM_NAMESPACE')
expect(namespaceEnv).toBeDefined()
@@ -199,7 +199,7 @@ describe('reconcile() — B1 integration: channel-reader Deployment shape (synce
await reconciler.reconcile(makeHost())
- const dep = getChannelReaderCreateBody(appsApi)
+ const dep = getChannelReaderConvergedBody(appsApi)
const envFrom = dep.spec?.template?.spec?.containers?.[0]?.envFrom ?? []
expect(envFrom).toContainEqual({ configMapRef: { name: 'clerum-channel-reader-config' } })
})
@@ -231,7 +231,7 @@ describe('reconcile() — B2 integration: replica preservation when CC cache is
// Return a "live replicas=1" response for the channel-reader read; for the host
// deployment read, return HCC-owned labels so reconcile() continues normally.
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
+ ({ name, namespace }: { name: string; namespace: string }) => {
if (name === 'channel-reader-alpha-host') {
return Promise.resolve({
metadata: {
@@ -250,6 +250,9 @@ describe('reconcile() — B2 integration: replica preservation when CC cache is
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
@@ -268,7 +271,12 @@ describe('reconcile() — B2 integration: replica preservation when CC cache is
await reconciler.reconcile(makeHost())
// The replace call body must carry replicas=1, not 0
- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
+ expect(
+ appsApi.replaceNamespacedDeployment.mock.calls.filter(
+ ([request]) => request.namespace === 'channels'
+ )
+ ).toHaveLength(1)
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
const replaceBody = getChannelReaderReplaceBody(appsApi)
expect(replaceBody.spec?.replicas).toBe(1)
})
@@ -285,12 +293,15 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
// channel-reader Deployment create succeeds; subsequent read (for status)
// returns readyReplicas=1.
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
+ ({ name, namespace }: { name: string; namespace: string }) => {
if (name === 'channel-reader-alpha-host') {
return Promise.resolve({
status: { readyReplicas: 1 },
spec: { replicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/host': 'alpha-host',
@@ -303,6 +314,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
@@ -328,7 +342,7 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
reconciler.setIsCommunicationChannelCacheSynced(() => true)
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
+ ({ name, namespace }: { name: string; namespace: string }) => {
if (name === 'channel-reader-alpha-host') {
const err = new Error('Not Found') as Error & { code?: number }
err.code = 404
@@ -338,6 +352,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
@@ -364,12 +381,15 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
reconciler.setIsCommunicationChannelCacheSynced(() => true)
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
+ ({ name, namespace }: { name: string; namespace: string }) => {
if (name === 'channel-reader-alpha-host') {
return Promise.resolve({
status: { readyReplicas: 0 },
spec: { replicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/host': 'alpha-host',
@@ -381,6 +401,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
@@ -409,13 +432,16 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
reconciler.setIsCommunicationChannelCacheSynced(() => true)
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
+ ({ name, namespace }: { name: string; namespace: string }) => {
if (name === 'channel-reader-alpha-host') {
// Deployment exists but scaled to 0 — no CCs.
return Promise.resolve({
status: { readyReplicas: 0 },
spec: { replicas: 0 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/host': 'alpha-host',
@@ -428,6 +454,9 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
@@ -455,18 +484,22 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
reconciler.setCountCommunicationChannels(() => 1)
reconciler.setIsCommunicationChannelCacheSynced(() => true)
+ let channelReads = 0
appsApi.readNamespacedDeployment.mockImplementation(
- ({ name }: { name: string; namespace: string }) => {
- if (name === 'channel-reader-alpha-host') {
+ ({ name, namespace }: { name: string; namespace: string }) => {
+ if (name === 'channel-reader-alpha-host' && ++channelReads > 1) {
return Promise.reject(new Error('api server unavailable'))
}
return Promise.resolve({
status: { readyReplicas: 1 },
metadata: {
+ name,
+ namespace,
+ uid: `uid-${name}`,
resourceVersion: '1',
labels: {
'clerum.io/managed-by': 'host-context-controller',
- 'clerum.io/host': name,
+ 'clerum.io/host': 'alpha-host',
},
},
})
@@ -474,6 +507,7 @@ describe('reconcile() — B8: channelReader status in HostRuntimeStatus', () =>
)
await reconciler.reconcile(makeHost())
+ expect(channelReads).toBe(2)
const status = reconciler.getStatus('alpha-host')
// mcp-host Deployment is still deployed+ready
diff --git a/host-context-controller/test/hostReconciler.lifecycle.test.ts b/host-context-controller/test/hostReconciler.lifecycle.test.ts
index 7fceb76a3..a87a002e1 100644
--- a/host-context-controller/test/hostReconciler.lifecycle.test.ts
+++ b/host-context-controller/test/hostReconciler.lifecycle.test.ts
@@ -418,7 +418,8 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {
await (reconciler as any).ensureDeployment(host, [], 'revision-a')
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledOnce()
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledOnce()
expect(appsApi.readNamespacedDeployment).toHaveBeenCalledWith({
namespace: 'mcp-host',
name: 'chatllm',
@@ -628,8 +629,11 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {
'CLERUM_STATELESS_LIFECYCLE'
)
- appsApi.createNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
- appsApi.readNamespacedDeployment.mockResolvedValueOnce(runningStateful)
+ appsApi.replaceNamespacedDeployment.mockClear()
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ request.name === stateful.name ? Promise.resolve(runningStateful) : readDeployment(request)
+ )
const stateless = makeStatelessHost({ name: 'transition-host' })
customApi.getNamespacedCustomObject.mockResolvedValue({
@@ -640,9 +644,11 @@ describe('HostReconciler ensureDeployment — idempotent replacement', () => {
await reconciler.reconcile(stateless)
- expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
- const replacement = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
- .body as k8s.V1Deployment
+ const hostReplacements = appsApi.replaceNamespacedDeployment.mock.calls.filter(
+ ([request]) => request.name === stateful.name
+ )
+ expect(hostReplacements).toHaveLength(1)
+ const replacement = hostReplacements[0][0].body as k8s.V1Deployment
expect(replacement.spec?.replicas).toBe(1)
expect(replacement.spec?.template.spec?.priorityClassName).toBe('clerum-interactive-host')
expect(
@@ -672,6 +678,17 @@ describe('HostReconciler stateless lifecycle — env injection', () => {
status: { lifecycle: { state: 'active', wakeHandledGeneration: 0 } },
})
+ // Model a genuinely new principal Deployment, then expose its created state.
+ let principalCreated = false
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request => {
+ if (request.name === stateless.name && !principalCreated) return Promise.reject({ code: 404 })
+ return readDeployment(request)
+ })
+ appsApi.createNamespacedDeployment.mockImplementation(async request => {
+ if (request.body.metadata?.name === stateless.name) principalCreated = true
+ return request.body
+ })
await reconciler.reconcile(stateless)
const creates = appsApi.createNamespacedDeployment.mock.calls.filter(
@@ -938,7 +955,7 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
it('propagates the principal Deployment mutation failure so lifecycle convergence can retry', async () => {
const { reconciler, appsApi } = createReconciler()
const failure = Object.assign(new Error('Deployment API unavailable'), { code: 503 })
- appsApi.createNamespacedDeployment.mockImplementation(
+ appsApi.replaceNamespacedDeployment.mockImplementation(
async ({ body }: { body: k8s.V1Deployment }) => {
if (body.metadata?.name === 'stateless-host') throw failure
return {}
@@ -948,12 +965,13 @@ describe('HostReconciler stateless lifecycle — rejection matrix', () => {
try {
await expect(reconciler.reconcile(makeStatelessHost())).rejects.toBe(failure)
+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
} finally {
errorSpy.mockRestore()
}
})
- it('rechecks channel policy after a Deployment create conflict before replace', async () => {
+ it('rechecks channel policy when the Deployment existence read observes a late channel', async () => {
let channelCount = 0
let hostDeploymentReads = 0
const { reconciler, appsApi, customApi } = createReconciler({
diff --git a/host-context-controller/test/hostReconciler.test.ts b/host-context-controller/test/hostReconciler.test.ts
index f0dd97d86..93d43ebda 100644
--- a/host-context-controller/test/hostReconciler.test.ts
+++ b/host-context-controller/test/hostReconciler.test.ts
@@ -244,6 +244,15 @@ describe('HostReconciler', () => {
if (name === 'host-alpha-host-mcp-host-runtime-tokens') return Promise.resolve(firstWrite)
return Promise.resolve({ metadata: { resourceVersion: '1' }, data: {} })
})
+ const channelReader = appsApi.replaceNamespacedDeployment.mock.calls.find(
+ ([request]) => request.name === 'channel-reader-alpha-host'
+ )![0].body
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request =>
+ request.name === 'channel-reader-alpha-host'
+ ? Promise.resolve(channelReader)
+ : readDeployment(request)
+ )
coreApi.replaceNamespacedSecret.mockClear()
appsApi.replaceNamespacedDeployment.mockClear()
@@ -280,7 +289,7 @@ describe('HostReconciler', () => {
).toBe(true)
})
- it('creates deployment, service, and pvc for valid host', async () => {
+ it('converges existing deployment and service and creates pvc for valid host', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
const host = makeHost()
@@ -291,10 +300,12 @@ describe('HostReconciler', () => {
name: 'host-secret',
})
expect(coreApi.createNamespacedPersistentVolumeClaim).toHaveBeenCalledTimes(1)
- // reconcile creates two Services: the host (mcp-host ns) and its
+ // Reconcile updates two existing Services: the host (mcp-host ns) and its
// channel-reader handoff Service (channels ns).
- expect(coreApi.createNamespacedService).toHaveBeenCalledTimes(2)
- expect(coreApi.createNamespacedService.mock.calls).toContainEqual([
+ expect(coreApi.readNamespacedService).toHaveBeenCalledTimes(2)
+ expect(coreApi.createNamespacedService).not.toHaveBeenCalled()
+ expect(coreApi.replaceNamespacedService).toHaveBeenCalledTimes(2)
+ expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
expect.objectContaining({
namespace: 'mcp-host',
body: expect.objectContaining({
@@ -302,7 +313,7 @@ describe('HostReconciler', () => {
}),
}),
])
- expect(coreApi.createNamespacedService.mock.calls).toContainEqual([
+ expect(coreApi.replaceNamespacedService.mock.calls).toContainEqual([
expect.objectContaining({
namespace: 'channels',
body: expect.objectContaining({
@@ -310,13 +321,14 @@ describe('HostReconciler', () => {
}),
}),
])
- // reconcile creates two Deployments: the host (mcp-host ns) and its
+ // Reconcile updates two existing Deployments: the host (mcp-host ns) and its
// channel-reader (channels ns).
- expect(appsApi.createNamespacedDeployment).toHaveBeenCalledTimes(2)
- expect(appsApi.createNamespacedDeployment.mock.calls).toContainEqual([
+ expect(appsApi.createNamespacedDeployment).not.toHaveBeenCalled()
+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(2)
+ expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
expect.objectContaining({ namespace: 'mcp-host' }),
])
- expect(appsApi.createNamespacedDeployment.mock.calls).toContainEqual([
+ expect(appsApi.replaceNamespacedDeployment.mock.calls).toContainEqual([
expect.objectContaining({ namespace: 'channels' }),
])
expect(reconciler.getStatus('alpha-host')).toMatchObject({ deployed: true, ready: true })
@@ -564,7 +576,7 @@ describe('HostReconciler', () => {
])
)
- const channelReaderCreate = appsApi.createNamespacedDeployment.mock.calls.find(([arg]) => {
+ const channelReaderCreate = appsApi.replaceNamespacedDeployment.mock.calls.find(([arg]) => {
const body = arg.body as k8s.V1Deployment
return body.metadata?.name === 'channel-reader-alpha-host'
})
@@ -697,7 +709,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host-desktop:latest')
})
@@ -707,7 +719,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ browser: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host-desktop:latest')
})
@@ -717,7 +729,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const ports = call.body.spec.template.spec.containers[0].ports
expect(ports).toHaveLength(2)
expect(ports[0]).toMatchObject({ name: 'http', containerPort: 8080 })
@@ -729,7 +741,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = coreApi.createNamespacedService.mock.calls[0][0]
+ const call = coreApi.replaceNamespacedService.mock.calls[0][0]
const ports = call.body.spec.ports
expect(ports).toHaveLength(2)
expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
@@ -741,7 +753,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true, browser: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))
expect(envMap.get('CLERUM_DESKTOP_X11')).toBe('true')
@@ -756,7 +768,7 @@ describe('HostReconciler — desktop support', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const envMap = new Map(env.map((e: { name: string; value?: string }) => [e.name, e.value]))
@@ -770,7 +782,7 @@ describe('HostReconciler — desktop support', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const env = call.body.spec.template.spec.containers[0].env
const controlEnv = env.find(
(e: { name: string }) => e.name === 'MCP_HOST_WORKFLOW_CONTROL_TOKEN'
@@ -787,7 +799,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const secCtx = call.body.spec.template.spec.securityContext
expect(secCtx.runAsNonRoot).toBe(false)
expect(secCtx.seccompProfile).toEqual({ type: 'RuntimeDefault' })
@@ -798,7 +810,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const startupProbe = call.body.spec.template.spec.containers[0].startupProbe
expect(startupProbe.initialDelaySeconds).toBe(30)
expect(startupProbe.failureThreshold).toBe(120)
@@ -809,7 +821,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeDesktopHost({ x11: true })
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const resources = call.body.spec.template.spec.containers[0].resources
expect(resources.requests.memory).toBe('256Mi')
expect(resources.requests.cpu).toBe('250m')
@@ -895,7 +907,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeHost()
await reconciler.reconcile(host)
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.image).toBe('clerum/mcp-host:0.6.0')
expect(container.resources.requests.memory).toBe('128Mi')
@@ -914,7 +926,7 @@ describe('HostReconciler — desktop support', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const secCtx = call.body.spec.template.spec.securityContext
expect(secCtx.runAsNonRoot).toBe(true)
// 1001 = the mcp-host image's baked-in `nodejs` user; matches the
@@ -931,7 +943,7 @@ describe('HostReconciler — desktop support', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const call = appsApi.createNamespacedDeployment.mock.calls[0][0]
+ const call = appsApi.replaceNamespacedDeployment.mock.calls[0][0]
const container = call.body.spec.template.spec.containers[0]
expect(container.name).toBe('mcp-host')
expect(container.securityContext).toMatchObject({
@@ -953,7 +965,7 @@ describe('HostReconciler — desktop support', () => {
const host = makeHost()
await reconciler.reconcile(host)
- const call = coreApi.createNamespacedService.mock.calls[0][0]
+ const call = coreApi.replaceNamespacedService.mock.calls[0][0]
const ports = call.body.spec.ports
expect(ports).toHaveLength(1)
expect(ports[0]).toMatchObject({ name: 'http', port: 8080 })
@@ -1015,7 +1027,7 @@ describe('HostReconciler — per-Host RBAC scaffolding', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const deployment = appsApi.createNamespacedDeployment.mock.calls[0][0].body
+ const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
expect(deployment.spec.template.spec.serviceAccountName).toBe('host-alpha-host-sa')
})
@@ -1023,7 +1035,7 @@ describe('HostReconciler — per-Host RBAC scaffolding', () => {
const { reconciler, appsApi } = createReconciler()
await reconciler.reconcile(makeHost())
- const deployment = appsApi.createNamespacedDeployment.mock.calls[0][0].body
+ const deployment = appsApi.replaceNamespacedDeployment.mock.calls[0][0].body
const env = deployment.spec.template.spec.containers[0].env as Array<{
name: string
value?: string
@@ -1292,6 +1304,7 @@ describe('reconcileChannelReaderDeployment', () => {
it('creates Deployment when absent (createNamespacedDeployment succeeds)', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockResolvedValue({})
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(makeHost({ name: 'a' }))
@@ -1304,7 +1317,7 @@ describe('reconcileChannelReaderDeployment', () => {
})
})
- it('replaces Deployment on 409 conflict (drift)', async () => {
+ it('replaces a present Deployment with drift without creating', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
appsApi.readNamespacedDeployment.mockResolvedValue({
@@ -1456,16 +1469,13 @@ describe('reconcileChannelReaderDeployment', () => {
converged.metadata = { ...converged.metadata, resourceVersion: '43' }
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
- appsApi.readNamespacedDeployment
- .mockResolvedValueOnce(stale)
- .mockResolvedValueOnce(stale)
- .mockResolvedValueOnce(converged)
+ appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(converged)
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
await (reconciler as any).reconcileChannelReaderDeployment(host)
- expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(3)
+ expect(appsApi.readNamespacedDeployment).toHaveBeenCalledTimes(2)
expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledOnce()
})
@@ -1485,10 +1495,7 @@ describe('reconcileChannelReaderDeployment', () => {
}
appsApi.createNamespacedDeployment.mockRejectedValue({ code: 409 })
- appsApi.readNamespacedDeployment
- .mockResolvedValueOnce(stale)
- .mockResolvedValueOnce(stale)
- .mockResolvedValueOnce(foreign)
+ appsApi.readNamespacedDeployment.mockResolvedValueOnce(stale).mockResolvedValueOnce(foreign)
appsApi.replaceNamespacedDeployment.mockRejectedValueOnce({ code: 409 })
coreApi.readNamespacedSecret.mockRejectedValue({ code: 404 })
@@ -1501,6 +1508,7 @@ describe('reconcileChannelReaderDeployment', () => {
it('reads existing Secret revision and writes annotation on initial create', async () => {
const { reconciler, appsApi, coreApi } = createReconciler()
+ appsApi.readNamespacedDeployment.mockRejectedValueOnce({ code: 404 })
appsApi.createNamespacedDeployment.mockResolvedValue({})
coreApi.readNamespacedSecret.mockResolvedValue({
data: { 'telegram-bot-token': Buffer.from('tok-1').toString('base64') },
@@ -1557,7 +1565,7 @@ describe('reconcile / reconcileDelete with channel-reader', () => {
.spyOn(reconciler as any, 'reconcileChannelReaderDeployment')
.mockResolvedValue(undefined)
await reconciler.reconcile(makeHost({ name: 'a' }))
- expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }))
+ expect(spy).toHaveBeenCalledWith(expect.objectContaining({ name: 'a' }), expect.any(Function))
})
it('reconcileDelete(name, ns) deletes channel-reader-<name> Deployment in channels ns', async () => {
diff --git a/host-context-controller/test/hostReconciler.wake.test.ts b/host-context-controller/test/hostReconciler.wake.test.ts
index 4fc7ae2aa..4c8fcb062 100644
--- a/host-context-controller/test/hostReconciler.wake.test.ts
+++ b/host-context-controller/test/hostReconciler.wake.test.ts
@@ -510,8 +510,18 @@ describe('HostReconciler wake fast-path — durability and restart behavior', ()
it('Deployment recreated while suspended stays at replicas=0 when the wake is already handled', async () => {
const { reconciler, appsApi } = createReconciler()
- // The mock create succeeds (no 409), i.e. the Deployment does not exist
- // yet — exactly the "recreated while suspended" case.
+ // The principal Deployment is absent until this reconcile creates it.
+ let principalCreated = false
+ const readDeployment = appsApi.readNamespacedDeployment.getMockImplementation()!
+ appsApi.readNamespacedDeployment.mockImplementation(request => {
+ if (request.name === 'stateless-host' && !principalCreated)
+ return Promise.reject({ code: 404 })
+ return readDeployment(request)
+ })
+ appsApi.createNamespacedDeployment.mockImplementation(async request => {
+ if (request.body.metadata?.name === 'stateless-host') principalCreated = true
+ return request.body
+ })
await reconciler.reconcile(
makeWakeHost({ wakeRequested: '2', state: 'suspended', wakeHandledGeneration: 2 })
)