Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): skip no-op gfsc-reader Deployment replaces

Alfredo Lopezd50f97532909
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index ac10a721e..85c4ed718 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.337",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.337",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index 921d081c5..1c0ad4de2 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.336",
+ "version": "0.1.337",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
index 19cbc2573..7524b664d 100644
--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts
@@ -123,7 +123,7 @@ describe('applyNetworkPolicy no-op gate', () => {
await expect(
applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())
- ).resolves.toBeUndefined()
+ ).resolves.toBe('missing')
expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()
expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
@@ -199,7 +199,7 @@ describe('applyNetworkPolicy no-op gate', () => {
'[NetPol]',
mutationAllowed
)
- ).resolves.toBeUndefined()
+ ).resolves.toBe('not_allowed')
expect(mutationAllowed).toHaveBeenCalledOnce()
expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()
diff --git a/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts b/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts
index b0242b642..d2fbc1d4c 100644
--- a/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts
+++ b/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts
@@ -9,9 +9,10 @@ const GATED = new Set([
'reconciler.ts',
'sharedFileSystemReconciler.ts',
'llmHookReconciler.ts',
+ 'k8s/gfsK8sApi.ts',
])
-const EXEMPT = new Set(['k8s/gfsK8sApi.ts', 'statelessLifecycleExecutor.ts'])
+const EXEMPT = new Set(['statelessLifecycleExecutor.ts'])
/** Any call, including `replace(params)` — not only inline `({ ... })`. */
const CALL = /replaceNamespacedDeployment\s*\(/g
@@ -84,6 +85,11 @@ function isDeploymentGated(block: string): boolean {
)
}
+/** Replica-only scale in K8sGfsApi is not a template replace; it is not a no-op gate. */
+function isReplicaScaleReplace(block: string): boolean {
+ return /replicas/.test(block) && /mergeExisting/.test(block) && !isDeploymentGated(block)
+}
+
describe('Deployment writer inventory', () => {
it('every replaceNamespacedDeployment call is gated or explicitly exempt', () => {
const hits: Array<{ rel: string; count: number; gated: boolean }> = []
@@ -119,6 +125,7 @@ describe('Deployment writer inventory', () => {
block,
`${rel} call at index ${index} is not inside replaceWithConflictRetry`
).not.toBeNull()
+ if (isReplicaScaleReplace(block ?? '')) continue
expect(
isDeploymentGated(block ?? ''),
`${rel} call at index ${index} is missing an isUpToDate deploymentMatchesDesired gate`
diff --git a/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.json b/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.json
new file mode 100644
index 000000000..95d1dd6f3
--- /dev/null
+++ b/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.json
@@ -0,0 +1,281 @@
+{
+ "apiVersion": "apps/v1",
+ "kind": "Deployment",
+ "metadata": {
+ "annotations": {
+ "deployment.kubernetes.io/revision": "511",
+ "clerum.io/gfsc-template-hash": "504c150f07082276"
+ },
+ "labels": {
+ "clerum.io/globalfilesystem": "gfs",
+ "clerum.io/managed-by": "host-context-controller"
+ },
+ "name": "gfsc-reader",
+ "namespace": "gfs"
+ },
+ "spec": {
+ "progressDeadlineSeconds": 600,
+ "replicas": 2,
+ "revisionHistoryLimit": 10,
+ "selector": {
+ "matchLabels": {
+ "app": "gfs-controller",
+ "clerum.io/gfsc-role": "reader"
+ }
+ },
+ "strategy": {
+ "rollingUpdate": {
+ "maxSurge": 1,
+ "maxUnavailable": 0
+ },
+ "type": "RollingUpdate"
+ },
+ "template": {
+ "metadata": {
+ "annotations": {
+ "clerum.io/gfsc-template-hash": "504c150f07082276"
+ },
+ "labels": {
+ "app": "gfs-controller",
+ "clerum.io/gfsc-role": "reader"
+ }
+ },
+ "spec": {
+ "affinity": {
+ "podAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": [
+ {
+ "labelSelector": {
+ "matchLabels": {
+ "app": "gfs-controller",
+ "clerum.io/gfsc-role": "writer"
+ }
+ },
+ "topologyKey": "kubernetes.io/hostname"
+ }
+ ]
+ }
+ },
+ "automountServiceAccountToken": false,
+ "containers": [
+ {
+ "env": [
+ {
+ "name": "GFS_PORT",
+ "value": "8087"
+ },
+ {
+ "name": "GFS_STORAGE_PATH",
+ "value": "/data/gfs"
+ },
+ {
+ "name": "GFS_STORAGE_ROLE",
+ "value": "reader"
+ },
+ {
+ "name": "GFS_DRIVE_NAME",
+ "value": "main"
+ },
+ {
+ "name": "GFS_TOKEN_AUDIENCE",
+ "value": "gfs-controller"
+ },
+ {
+ "name": "GFS_JWT_PUBLIC_KEY",
+ "valueFrom": {
+ "configMapKeyRef": {
+ "key": "jwt-public-key",
+ "name": "synthetic-gfs-config"
+ }
+ }
+ },
+ {
+ "name": "GFS_PG_CONNECTION_STRING",
+ "valueFrom": {
+ "secretKeyRef": {
+ "key": "connection-string",
+ "name": "synthetic-reader-db"
+ }
+ }
+ },
+ {
+ "name": "GFS_MAX_WRITE_BODY_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_PROTOCOL_MAX_FILE_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_PRODUCT_MAX_FILE_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_FILE_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_PREFERRED_CHUNK_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_CHUNK_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MIN_PART_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_PART_COUNT",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_SESSION_TTL_MS",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_COMPLETED_RECEIPT_TTL_MS",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_STALE_PART_LEASE_MS",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_ACTIVE_PER_SUBJECT",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_ACTIVE_GLOBAL",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_CONCURRENT_PARTS_PER_SESSION",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_CONCURRENT_PART_STREAMS_GLOBAL",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MAX_CONCURRENT_FINALIZATIONS",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_MIN_FREE_BYTES",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_PART_TIMEOUT_MS",
+ "value": "1"
+ },
+ {
+ "name": "GFS_UPLOAD_FINALIZE_TIMEOUT_MS",
+ "value": "1"
+ }
+ ],
+ "image": "registry.example.com/gfs-controller:synthetic",
+ "imagePullPolicy": "IfNotPresent",
+ "livenessProbe": {
+ "failureThreshold": 3,
+ "httpGet": {
+ "path": "/healthz",
+ "port": "http",
+ "scheme": "HTTP"
+ },
+ "initialDelaySeconds": 10,
+ "periodSeconds": 30,
+ "successThreshold": 1,
+ "timeoutSeconds": 2
+ },
+ "name": "gfsc",
+ "ports": [
+ {
+ "containerPort": 8087,
+ "name": "http",
+ "protocol": "TCP"
+ }
+ ],
+ "readinessProbe": {
+ "failureThreshold": 3,
+ "httpGet": {
+ "path": "/readyz",
+ "port": "http",
+ "scheme": "HTTP"
+ },
+ "initialDelaySeconds": 2,
+ "periodSeconds": 5,
+ "successThreshold": 1,
+ "timeoutSeconds": 2
+ },
+ "resources": {
+ "limits": {
+ "cpu": "500m",
+ "memory": "512Mi"
+ },
+ "requests": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ }
+ },
+ "securityContext": {
+ "allowPrivilegeEscalation": false,
+ "capabilities": {
+ "drop": ["ALL"]
+ },
+ "readOnlyRootFilesystem": true
+ },
+ "startupProbe": {
+ "failureThreshold": 30,
+ "httpGet": {
+ "path": "/readyz",
+ "port": "http",
+ "scheme": "HTTP"
+ },
+ "initialDelaySeconds": 2,
+ "periodSeconds": 2,
+ "successThreshold": 1,
+ "timeoutSeconds": 1
+ },
+ "terminationMessagePath": "/dev/termination-log",
+ "terminationMessagePolicy": "File",
+ "volumeMounts": [
+ {
+ "mountPath": "/data/gfs",
+ "name": "drive",
+ "readOnly": true
+ }
+ ]
+ }
+ ],
+ "dnsPolicy": "ClusterFirst",
+ "restartPolicy": "Always",
+ "schedulerName": "default-scheduler",
+ "securityContext": {
+ "fsGroup": 1000,
+ "fsGroupChangePolicy": "OnRootMismatch",
+ "runAsGroup": 1000,
+ "runAsNonRoot": true,
+ "runAsUser": 1000,
+ "seccompProfile": {
+ "type": "RuntimeDefault"
+ }
+ },
+ "terminationGracePeriodSeconds": 30,
+ "volumes": [
+ {
+ "name": "drive",
+ "persistentVolumeClaim": {
+ "claimName": "gfs-drive"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
index b7688dc2b..5ddd0eec1 100644
--- a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
+++ b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts
@@ -171,7 +171,7 @@ describe('replaceWithConflictRetry order and retry', () => {
read,
replace,
})
- ).resolves.toBeUndefined()
+ ).resolves.toBe('missing')
expect(read).toHaveBeenCalledTimes(2)
expect(validateExisting).toHaveBeenCalledOnce()
diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.ts
index d2fd7965b..6cb69d8f0 100644
--- a/host-context-controller/src/gfsReconciler.test.ts
+++ b/host-context-controller/src/gfsReconciler.test.ts
@@ -1,8 +1,11 @@
-import { describe, expect, it } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'
import { GfsFactoryConfig } from './k8s/gfsFactory'
+import { K8sGfsApi } from './k8s/gfsK8sApi'
+import { hccLogger } from './logger'
import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'
+import type { ResourceApplyResult } from './utils'
const config: GfsFactoryConfig = {
gfsNamespace: 'gfs',
@@ -48,38 +51,51 @@ class FakeApi implements GfsK8sApi {
writerAvailabilitySequence: boolean[] = []
failPvc = false
statusFailuresRemaining = 0
+ applyOutcome: 'created' | 'replaced' | 'up_to_date' = 'replaced'
+ readerApplyOutcome?: 'created' | 'replaced' | 'up_to_date'
- async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {
+ async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<ResourceApplyResult> {
if (this.failPvc) throw new Error('simulated PVC apply failure')
this.pvcs.push(pvc.metadata?.name ?? '')
this.operations.push(`pvc/${pvc.metadata?.name ?? ''}`)
+ return this.applyOutcome
}
async deploymentNeedsUpdate(dep: k8s.V1Deployment): Promise<boolean> {
return dep.metadata?.name === 'gfsc-writer' ? this.writerNeedsUpdate : false
}
- async scaleDeployment(name: string, _namespace: string, replicas: number): Promise<void> {
+ async scaleDeployment(
+ name: string,
+ _namespace: string,
+ replicas: number
+ ): Promise<ResourceApplyResult> {
this.operations.push(`scale/${name}/${replicas}`)
+ return this.applyOutcome === 'up_to_date' ? 'up_to_date' : 'replaced'
}
- async applyDeployment(dep: k8s.V1Deployment): Promise<void> {
+ async applyDeployment(dep: k8s.V1Deployment): Promise<ResourceApplyResult> {
const name = dep.metadata?.name ?? ''
this.deployments.push(name)
this.deploymentManifests.push(dep)
this.operations.push(`deploy/${name}`)
+ if (name === 'gfsc-reader' && this.readerApplyOutcome) return this.readerApplyOutcome
+ return this.applyOutcome
}
- async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget): Promise<void> {
+ async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget): Promise<ResourceApplyResult> {
const name = pdb.metadata?.name ?? ''
this.pdbs.push(name)
this.operations.push(`pdb/${name}`)
+ return this.applyOutcome
}
- async applyService(svc: k8s.V1Service): Promise<void> {
+ async applyService(svc: k8s.V1Service): Promise<ResourceApplyResult> {
const name = svc.metadata?.name ?? ''
this.services.push(name)
this.operations.push(`svc/${name}`)
+ return this.applyOutcome
}
- async applyNetworkPolicy(np: k8s.V1NetworkPolicy): Promise<void> {
+ async applyNetworkPolicy(np: k8s.V1NetworkPolicy): Promise<ResourceApplyResult> {
const name = np.metadata?.name ?? ''
this.netpols.push(name)
this.operations.push(`np/${name}`)
+ return this.applyOutcome
}
async isDeploymentAvailable(name: string): Promise<boolean> {
if (name === 'gfsc-writer' && this.writerAvailabilitySequence.length > 0) {
@@ -266,7 +282,9 @@ describe('GfsReconciler.reconcile', () => {
const reconciler = new GfsReconciler(api, config)
await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')
- await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()
+ await expect(reconciler.reconcile(gfs)).resolves.toEqual(
+ expect.objectContaining({ objects: expect.any(Number) })
+ )
expect(api.statuses).toHaveLength(1)
expect(api.statuses[0]?.phase).toBe('Ready')
@@ -296,6 +314,164 @@ describe('GfsReconciler.fullReconcile', () => {
})
})
+function createStoreBackedGfsApi() {
+ const store = new Map<string, Record<string, unknown>>()
+ const reads = { total: 0, secondPass: 0 }
+ const replaces = { deployment: 0, pdb: 0, secondPassDeployment: 0, secondPassPdb: 0 }
+ let pass = 1
+
+ const keyOf = (kind: string, name: string) => `${kind}/${name}`
+
+ const read =
+ (kind: string) =>
+ async ({ name }: { name: string }) => {
+ reads.total += 1
+ if (pass === 2) reads.secondPass += 1
+ const stored = store.get(keyOf(kind, name))
+ if (!stored) throw { code: 404 }
+ return structuredClone(stored)
+ }
+
+ const create =
+ (kind: string) =>
+ async ({ body }: { body: { metadata?: { name?: string } } }) => {
+ const name = body.metadata?.name ?? ''
+ const stored = structuredClone(body) as Record<string, unknown>
+ const metadata = {
+ ...((stored.metadata as Record<string, unknown> | undefined) ?? {}),
+ resourceVersion: '1',
+ ...(kind === 'Deployment'
+ ? {
+ generation: 1,
+ annotations: {
+ ...(((stored.metadata as { annotations?: Record<string, string> } | undefined)
+ ?.annotations ?? {}) as Record<string, string>),
+ 'deployment.kubernetes.io/revision': '1',
+ },
+ }
+ : {}),
+ }
+ stored.metadata = metadata
+ if (kind === 'Deployment') {
+ stored.status = { availableReplicas: 1, observedGeneration: 1 }
+ }
+ store.set(keyOf(kind, name), stored)
+ return structuredClone(stored)
+ }
+
+ const replace =
+ (kind: string) =>
+ async ({ body }: { body: { metadata?: { name?: string } } }) => {
+ if (kind === 'Deployment') {
+ replaces.deployment += 1
+ if (pass === 2) replaces.secondPassDeployment += 1
+ }
+ if (kind === 'PodDisruptionBudget') {
+ replaces.pdb += 1
+ if (pass === 2) replaces.secondPassPdb += 1
+ }
+ const name = body.metadata?.name ?? ''
+ const stored = structuredClone(body) as Record<string, unknown>
+ stored.metadata = {
+ ...((stored.metadata as Record<string, unknown> | undefined) ?? {}),
+ resourceVersion: '2',
+ }
+ store.set(keyOf(kind, name), stored)
+ return structuredClone(stored)
+ }
+
+ const sdk = {
+ readNamespacedPersistentVolumeClaim: read('PersistentVolumeClaim'),
+ createNamespacedPersistentVolumeClaim: create('PersistentVolumeClaim'),
+ readNamespacedService: read('Service'),
+ createNamespacedService: create('Service'),
+ readNamespacedDeployment: read('Deployment'),
+ createNamespacedDeployment: create('Deployment'),
+ replaceNamespacedDeployment: replace('Deployment'),
+ readNamespacedPodDisruptionBudget: read('PodDisruptionBudget'),
+ createNamespacedPodDisruptionBudget: create('PodDisruptionBudget'),
+ replaceNamespacedPodDisruptionBudget: replace('PodDisruptionBudget'),
+ readNamespacedNetworkPolicy: read('NetworkPolicy'),
+ createNamespacedNetworkPolicy: create('NetworkPolicy'),
+ replaceNamespacedNetworkPolicy: replace('NetworkPolicy'),
+ patchNamespacedCustomObjectStatus: async () => ({}),
+ }
+
+ return {
+ api: new K8sGfsApi(
+ sdk as unknown as k8s.CoreV1Api,
+ sdk as unknown as k8s.AppsV1Api,
+ sdk as unknown as k8s.NetworkingV1Api,
+ sdk as unknown as k8s.PolicyV1Api,
+ sdk as unknown as k8s.CustomObjectsApi
+ ),
+ reads,
+ replaces,
+ beginSecondPass() {
+ pass = 2
+ },
+ }
+}
+
+describe('GfsReconciler Deployment gate across two reconciles (T7)', () => {
+ it('T7: second reconcile does not write Deployment or PDB and still reads', async () => {
+ const cluster = createStoreBackedGfsApi()
+ const reconciler = new GfsReconciler(cluster.api, config, undefined, 0, 0)
+ await reconciler.reconcile(gfs)
+ cluster.beginSecondPass()
+ await reconciler.reconcile({ ...gfs, status: { phase: 'Ready' } })
+ expect(cluster.reads.secondPass).toBeGreaterThan(0)
+ expect(cluster.replaces.secondPassDeployment).toBe(0)
+ expect(cluster.replaces.secondPassPdb).toBe(0)
+ })
+})
+
+describe('GfsReconciler resync pass log (D3)', () => {
+ beforeEach(() => {
+ vi.restoreAllMocks()
+ })
+
+ function resyncCalls() {
+ return vi
+ .mocked(hccLogger.info)
+ .mock.calls.filter(([message]) => message === 'Resync pass completed')
+ }
+
+ it('D3: an already-synced GFS pass logs writes 0 and skips equal to objects', async () => {
+ const api = new FakeApi()
+ api.writerNeedsUpdate = false
+ api.applyOutcome = 'up_to_date'
+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})
+ await new GfsReconciler(api, config).fullReconcile([gfs])
+ expect(info).toHaveBeenCalled()
+ const line = resyncCalls().at(-1)
+ expect(line?.[0]).toBe('Resync pass completed')
+ expect(line?.[1]).toEqual(
+ expect.objectContaining({
+ scope: 'GFS',
+ writes: 0,
+ objects: expect.any(Number),
+ skips: expect.any(Number),
+ passMs: expect.any(Number),
+ })
+ )
+ const fields = line?.[1] as { objects: number; skips: number }
+ expect(fields.skips).toBe(fields.objects)
+ expect(fields.objects).toBeGreaterThan(0)
+ })
+
+ it('D3: a drifted reader logs writes 1', async () => {
+ const api = new FakeApi()
+ api.writerNeedsUpdate = false
+ api.applyOutcome = 'up_to_date'
+ api.readerApplyOutcome = 'replaced'
+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})
+ await new GfsReconciler(api, config).fullReconcile([gfs])
+ const line = resyncCalls().at(-1)
+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'GFS', writes: 1 }))
+ })
+})
+
describe('GfsReconciler.reconcileDelete', () => {
it('retains the PVC by default (tears down broker, keeps bytes)', async () => {
const api = new FakeApi()
diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.ts
index 0462a12e6..b6adece9a 100644
--- a/host-context-controller/src/gfsReconciler.ts
+++ b/host-context-controller/src/gfsReconciler.ts
@@ -36,20 +36,44 @@ import {
writerDeploymentName,
writerServiceName,
} from './k8s/gfsFactory'
+import { hccLogger } from './logger'
import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'
+import type { ResourceApplyResult } from './utils'
/**
* Minimal Kubernetes surface the reconciler needs. PVCs are create-if-absent
* (their spec is immutable post-bind); everything else is create-or-replace.
*/
+export interface GfsResyncStats {
+ objects: number
+ writes: number
+ skips: number
+}
+
+function emptyGfsResyncStats(): GfsResyncStats {
+ return { objects: 0, writes: 0, skips: 0 }
+}
+
+function recordGfsApply(
+ stats: GfsResyncStats,
+ result: ResourceApplyResult | 'writer_hash_skip'
+): void {
+ stats.objects += 1
+ if (result === 'created' || result === 'replaced') stats.writes += 1
+ else if (result === 'up_to_date' || result === 'writer_hash_skip') stats.skips += 1
+}
+
export interface GfsK8sApi {
- applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<void>
+ applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<ResourceApplyResult>
deploymentNeedsUpdate(dep: k8s.V1Deployment, namespace: string): Promise<boolean>
- scaleDeployment(name: string, namespace: string, replicas: number): Promise<void>
- applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<void>
- applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget, namespace: string): Promise<void>
- applyService(svc: k8s.V1Service, namespace: string): Promise<void>
- applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<void>
+ scaleDeployment(name: string, namespace: string, replicas: number): Promise<ResourceApplyResult>
+ applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<ResourceApplyResult>
+ applyPodDisruptionBudget(
+ pdb: k8s.V1PodDisruptionBudget,
+ namespace: string
+ ): Promise<ResourceApplyResult>
+ applyService(svc: k8s.V1Service, namespace: string): Promise<ResourceApplyResult>
+ applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<ResourceApplyResult>
/** True when the named Deployment reports at least one available replica. */
isDeploymentAvailable(name: string, namespace: string): Promise<boolean>
deleteDeployment(name: string, namespace: string): Promise<void>
@@ -116,29 +140,41 @@ export class GfsReconciler {
await this.api.patchStatus(gfs.name, gfs.namespace, status)
}
- async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {
+ async reconcile(gfs: GlobalFileSystemCRD): Promise<GfsResyncStats> {
+ const stats = emptyGfsResyncStats()
const ns = this.config.gfsNamespace
const writerDeployment = buildDeployment(gfs, this.config, 'writer')
const readerDeployment = buildDeployment(gfs, this.config, 'reader')
const writerName = writerDeploymentName(gfs)
const readerName = readerDeploymentName(gfs)
- await this.api.applyPvc(buildPvc(gfs, this.config), ns)
+ recordGfsApply(stats, await this.api.applyPvc(buildPvc(gfs, this.config), ns))
const writerNeedsUpdate = await this.api.deploymentNeedsUpdate(writerDeployment, ns)
if (writerNeedsUpdate) {
// With standard-rwo, reader pods keep RO mounts on the writer's node.
// During a writer template rollout, scale them down before applying the
// writer so Kubernetes can attach the PVC to the writer's next node.
- await this.api.scaleDeployment(readerName, ns, 0)
+ recordGfsApply(stats, await this.api.scaleDeployment(readerName, ns, 0))
}
- await this.api.applyPodDisruptionBudget(buildPodDisruptionBudget(gfs, this.config), ns)
- await this.api.applyService(buildService(gfs, this.config), ns)
- await this.api.applyService(buildWriterService(gfs, this.config), ns)
- await this.api.applyNetworkPolicy(buildIngressNetworkPolicy(gfs, this.config), ns)
- await this.api.applyNetworkPolicy(buildEgressNetworkPolicy(gfs, this.config), ns)
+ recordGfsApply(
+ stats,
+ await this.api.applyPodDisruptionBudget(buildPodDisruptionBudget(gfs, this.config), ns)
+ )
+ recordGfsApply(stats, await this.api.applyService(buildService(gfs, this.config), ns))
+ recordGfsApply(stats, await this.api.applyService(buildWriterService(gfs, this.config), ns))
+ recordGfsApply(
+ stats,
+ await this.api.applyNetworkPolicy(buildIngressNetworkPolicy(gfs, this.config), ns)
+ )
+ recordGfsApply(
+ stats,
+ await this.api.applyNetworkPolicy(buildEgressNetworkPolicy(gfs, this.config), ns)
+ )
if (writerNeedsUpdate) {
- await this.api.applyDeployment(writerDeployment, ns)
+ recordGfsApply(stats, await this.api.applyDeployment(writerDeployment, ns))
+ } else {
+ recordGfsApply(stats, 'writer_hash_skip')
}
let writerReady = writerNeedsUpdate
@@ -148,11 +184,11 @@ export class GfsReconciler {
// A current writer template can still be unavailable after a node drain or
// pod eviction. Readers may retain RO mounts on the old node; release them
// so the single RWO writer can attach first, then restore readers below.
- await this.api.scaleDeployment(readerName, ns, 0)
+ recordGfsApply(stats, await this.api.scaleDeployment(readerName, ns, 0))
writerReady = await this.waitForWriterAvailable(gfs, ns)
}
if (writerReady) {
- await this.api.applyDeployment(readerDeployment, ns)
+ recordGfsApply(stats, await this.api.applyDeployment(readerDeployment, ns))
}
const phase = writerReady ? 'Ready' : 'Initializing'
@@ -168,6 +204,7 @@ export class GfsReconciler {
await this.seedClient.seedRootDirectories(gfs)
this.seeded.add(gfsKey(gfs))
}
+ return stats
}
/**
@@ -177,17 +214,29 @@ export class GfsReconciler {
* not a swallowed error: the failure is surfaced and the loop is idempotent.
*/
async fullReconcile(gfses: GlobalFileSystemCRD[]): Promise<void> {
+ const started = Date.now()
+ const totals = emptyGfsResyncStats()
for (const gfs of gfses) {
try {
- await this.reconcile(gfs)
+ const pass = await this.reconcile(gfs)
+ totals.objects += pass.objects
+ totals.writes += pass.writes
+ totals.skips += pass.skips
} catch (err) {
- console.error(
- `[gfsReconciler] reconcile failed for "${gfsKey(gfs)}": ${
- err instanceof Error ? err.message : String(err)
- }`
- )
+ hccLogger.error('reconcile failed', {
+ scope: 'GFS',
+ gfs: gfsKey(gfs),
+ err,
+ })
}
}
+ hccLogger.info('Resync pass completed', {
+ scope: 'GFS',
+ objects: totals.objects,
+ writes: totals.writes,
+ skips: totals.skips,
+ passMs: Date.now() - started,
+ })
}
async reconcileDelete(gfs: GlobalFileSystemCRD): Promise<void> {
diff --git a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
index 6dc85e953..7498b15db 100644
--- a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
+++ b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type * as k8s from '@kubernetes/client-node'
+import liveReaderFixture from '../__tests__/fixtures/629/gfsc-reader.deployment.json'
import livePdbFixture from '../__tests__/fixtures/629/gfsc-writer-pdb.json'
import { createsTotal, existenceReadsTotal, writeSkipsTotal } from '../metrics'
import { podDisruptionBudgetMatchesDesired } from '../utils'
@@ -34,7 +35,7 @@ const cases = [
},
{
kind: 'Deployment',
- updates: true,
+ updates: false,
body: {
kind: 'Deployment',
metadata,
@@ -200,7 +201,7 @@ describe.each(cases)('K8sGfsApi $kind read-first', resource => {
const h = harness(resource.kind)
h.read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce({ code: 404 })
h.create.mockRejectedValueOnce({ code: 409 })
- await expect(apply(h.api)).resolves.toBeUndefined()
+ await expect(apply(h.api)).resolves.toBe('missing')
expect(h.read).toHaveBeenCalledTimes(2)
expect(h.create).toHaveBeenCalledTimes(1)
expect(h.replace).not.toHaveBeenCalled()
@@ -345,3 +346,111 @@ describe('K8sGfsApi PodDisruptionBudget no-op gate (T4)', () => {
expect(podDisruptionBudgetMatchesDesired(desired, live)).toBe(false)
})
})
+
+const REVISION_ANNOTATION = 'deployment.kubernetes.io/revision'
+const RESTARTED_AT = 'kubectl.kubernetes.io/restartedAt'
+
+function desiredReaderFromFixture(): k8s.V1Deployment {
+ const desired = structuredClone(liveReaderFixture) as k8s.V1Deployment
+ if (desired.metadata?.annotations) {
+ delete desired.metadata.annotations[REVISION_ANNOTATION]
+ }
+ return desired
+}
+
+function liveReader(overrides: Partial<k8s.V1Deployment['spec']> = {}): k8s.V1Deployment {
+ const live = structuredClone(liveReaderFixture) as k8s.V1Deployment
+ return {
+ ...live,
+ metadata: {
+ ...live.metadata,
+ resourceVersion: '1',
+ uid: 'reader-uid',
+ creationTimestamp: new Date('2026-01-01T00:00:00Z'),
+ managedFields: [{ manager: 'kube-controller-manager', operation: 'Update' }],
+ },
+ spec: {
+ ...live.spec,
+ ...overrides,
+ },
+ status: { observedGeneration: 1, availableReplicas: live.spec?.replicas ?? 2 },
+ }
+}
+
+describe('K8sGfsApi Deployment no-op gate (T3, T8)', () => {
+ const namespace = 'gfs'
+
+ async function deploymentSkipCount() {
+ return (
+ (await writeSkipsTotal.get()).values.find(row => row.labels.kind === 'Deployment')?.value ?? 0
+ )
+ }
+
+ it('T3: skips replace when the live reader matches after revision merge', async () => {
+ const desired = desiredReaderFromFixture()
+ const equal = harness('Deployment')
+ equal.read.mockImplementation(async () => {
+ equal.events.push('GET')
+ return liveReader()
+ })
+ await equal.api.applyDeployment(desired, namespace)
+ expect(equal.read).toHaveBeenCalledTimes(1)
+ expect(equal.replace).toHaveBeenCalledTimes(0)
+ expect(await deploymentSkipCount()).toBe(1)
+ })
+
+ it('T3: writes when live replicas are 0 and restores the desired replica count', async () => {
+ const desired = desiredReaderFromFixture()
+ const scaled = harness('Deployment')
+ scaled.read.mockImplementation(async () => {
+ scaled.events.push('GET')
+ return liveReader({ replicas: 0 })
+ })
+ await scaled.api.applyDeployment(desired, namespace)
+ expect(scaled.read).toHaveBeenCalledTimes(1)
+ expect(scaled.replace).toHaveBeenCalledTimes(1)
+ const replaced = scaled.replace.mock.calls[0][0].body as k8s.V1Deployment
+ expect(replaced.spec?.replicas).toBe(desired.spec?.replicas)
+ expect(await deploymentSkipCount()).toBe(0)
+ })
+
+ it('T8: skips replace when live template has restartedAt and increments writeSkipsTotal', async () => {
+ const desired = desiredReaderFromFixture()
+ const restarted = harness('Deployment')
+ const live = liveReader()
+ const templateMeta = live.spec?.template?.metadata
+ if (!templateMeta) throw new Error('expected pod template metadata')
+ templateMeta.annotations = {
+ ...templateMeta.annotations,
+ [RESTARTED_AT]: '2026-09-16T20:33:45Z',
+ }
+ restarted.read.mockImplementation(async () => {
+ restarted.events.push('GET')
+ return live
+ })
+ await restarted.api.applyDeployment(desired, namespace)
+ expect(restarted.read).toHaveBeenCalledTimes(1)
+ expect(restarted.replace).toHaveBeenCalledTimes(0)
+ expect(await deploymentSkipCount()).toBe(1)
+ })
+
+ it('T3: retries a drifted reader PUT after 409 using a fresh resourceVersion', async () => {
+ const desired = desiredReaderFromFixture()
+ const h = harness('Deployment')
+ let version = 0
+ h.read.mockImplementation(async () => {
+ h.events.push('GET')
+ const live = liveReader({ replicas: 0 })
+ live.metadata = { ...live.metadata, resourceVersion: String(++version) }
+ return live
+ })
+ h.replace.mockImplementationOnce(async ({ body }) => {
+ h.events.push(`PUT:${body.metadata?.resourceVersion}`)
+ throw { code: 409 }
+ })
+ await h.api.applyDeployment(desired, namespace)
+ expect(h.events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])
+ expect(h.replace).toHaveBeenCalledTimes(2)
+ expect(h.create).not.toHaveBeenCalled()
+ })
+})
diff --git a/host-context-controller/src/k8s/gfsK8sApi.ts b/host-context-controller/src/k8s/gfsK8sApi.ts
index 681ec5a1b..a82e093e6 100644
--- a/host-context-controller/src/k8s/gfsK8sApi.ts
+++ b/host-context-controller/src/k8s/gfsK8sApi.ts
@@ -3,12 +3,15 @@ import type { GfsK8sApi } from '../gfsReconciler'
import { createsTotal } from '../metrics'
import type { GlobalFileSystemStatus } from '../types'
import {
+ type ResourceApplyResult,
applyNetworkPolicy,
+ deploymentMatchesDesired,
ensureResource,
getErrorCode,
observeCreate,
observeExistenceRead,
podDisruptionBudgetMatchesDesired,
+ preserveDeploymentAnnotations,
preserveObjectAnnotations,
replaceWithConflictRetry,
} from '../utils'
@@ -45,9 +48,12 @@ export class K8sGfsApi implements GfsK8sApi {
}
}
- async applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<void> {
+ async applyPvc(
+ pvc: k8s.V1PersistentVolumeClaim,
+ namespace: string
+ ): Promise<ResourceApplyResult> {
const name = pvc.metadata?.name ?? ''
- await ensureResource<k8s.V1PersistentVolumeClaim>({
+ return ensureResource<k8s.V1PersistentVolumeClaim>({
read: () =>
observeExistenceRead('PersistentVolumeClaim', () =>
this.coreApi.readNamespacedPersistentVolumeClaim({ name, namespace })
@@ -78,17 +84,21 @@ export class K8sGfsApi implements GfsK8sApi {
}
}
- async scaleDeployment(name: string, namespace: string, replicas: number): Promise<void> {
+ async scaleDeployment(
+ name: string,
+ namespace: string,
+ replicas: number
+ ): Promise<ResourceApplyResult> {
let existing: k8s.V1Deployment
try {
existing = await this.appsApi.readNamespacedDeployment({ name, namespace })
} catch (err) {
- if (getErrorCode(err) === 404) return
+ if (getErrorCode(err) === 404) return 'missing'
throw err
}
- if ((existing.spec?.replicas ?? 0) === replicas) return
+ if ((existing.spec?.replicas ?? 0) === replicas) return 'up_to_date'
- await replaceWithConflictRetry<k8s.V1Deployment>({
+ return replaceWithConflictRetry<k8s.V1Deployment>({
description: `deployment "${name}" scale in ${namespace}`,
logPrefix: LOG,
body: existing,
@@ -106,9 +116,9 @@ export class K8sGfsApi implements GfsK8sApi {
})
}
- async applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<void> {
+ async applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<ResourceApplyResult> {
const name = dep.metadata?.name ?? ''
- await ensureResource<k8s.V1Deployment>({
+ return ensureResource<k8s.V1Deployment>({
read: () =>
observeExistenceRead('Deployment', () =>
this.appsApi.readNamespacedDeployment({ name, namespace })
@@ -125,13 +135,18 @@ export class K8sGfsApi implements GfsK8sApi {
body: dep,
read,
replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace, body }),
+ mergeExisting: preserveDeploymentAnnotations,
+ isUpToDate: deploymentMatchesDesired,
}),
})
}
- async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget, namespace: string): Promise<void> {
+ async applyPodDisruptionBudget(
+ pdb: k8s.V1PodDisruptionBudget,
+ namespace: string
+ ): Promise<ResourceApplyResult> {
const name = pdb.metadata?.name ?? ''
- await ensureResource<k8s.V1PodDisruptionBudget>({
+ return ensureResource<k8s.V1PodDisruptionBudget>({
read: () =>
observeExistenceRead('PodDisruptionBudget', () =>
this.policyApi.readNamespacedPodDisruptionBudget({ name, namespace })
@@ -155,9 +170,9 @@ export class K8sGfsApi implements GfsK8sApi {
})
}
- async applyService(svc: k8s.V1Service, namespace: string): Promise<void> {
+ async applyService(svc: k8s.V1Service, namespace: string): Promise<ResourceApplyResult> {
const name = svc.metadata?.name ?? ''
- await ensureResource<k8s.V1Service>({
+ return ensureResource<k8s.V1Service>({
read: () =>
observeExistenceRead('Service', () =>
this.coreApi.readNamespacedService({ name, namespace })
@@ -173,8 +188,11 @@ export class K8sGfsApi implements GfsK8sApi {
})
}
- async applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<void> {
- await applyNetworkPolicy(this.networkingApi, np.metadata?.name ?? '', namespace, np, LOG)
+ async applyNetworkPolicy(
+ np: k8s.V1NetworkPolicy,
+ namespace: string
+ ): Promise<ResourceApplyResult> {
+ return applyNetworkPolicy(this.networkingApi, np.metadata?.name ?? '', namespace, np, LOG)
}
async isDeploymentAvailable(name: string, namespace: string): Promise<boolean> {
diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.ts
index 9a55f1462..ac83bd2e9 100644
--- a/host-context-controller/src/llmHookReconciler.test.ts
+++ b/host-context-controller/src/llmHookReconciler.test.ts
@@ -21,6 +21,7 @@ import {
podKeyResourceName,
referencedHookIds,
} from './llmHookReconciler'
+import { hccLogger } from './logger'
import { HostCRD, LlmHookCRD } from './types'
vi.mock('./config', () => ({
@@ -1032,4 +1033,134 @@ describe('LlmHookReconciler', () => {
expect(new Set(referencedHookIds(host))).toEqual(new Set(['x', 'y', 'x']))
expect(referencedHookIds(undefined)).toEqual([])
})
+
+ describe('fullReconcile resync pass log (D3)', () => {
+ function lastBody<T>(fn: { mock: { calls: unknown[][] } }): T | undefined {
+ const call = fn.mock.calls.at(-1)?.[0] as { body?: T } | undefined
+ return call?.body
+ }
+
+ function resyncCalls() {
+ return vi
+ .mocked(hccLogger.info)
+ .mock.calls.filter(([message]) => message === 'Resync pass completed')
+ }
+
+ async function seedMatchingHook() {
+ const hook = makeHook({
+ name: 'd3',
+ spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },
+ })
+ hooks.set('d3', hook)
+ await reconciler.fullReconcile([hook])
+ const dep =
+ lastBody<k8s.V1Deployment>(appsApi.replaceNamespacedDeployment) ??
+ lastBody<k8s.V1Deployment>(appsApi.createNamespacedDeployment)
+ const svc =
+ lastBody<k8s.V1Service>(coreApi.replaceNamespacedService) ??
+ lastBody<k8s.V1Service>(coreApi.createNamespacedService)
+ const np =
+ lastBody<k8s.V1NetworkPolicy>(networkingApi.replaceNamespacedNetworkPolicy) ??
+ lastBody<k8s.V1NetworkPolicy>(networkingApi.createNamespacedNetworkPolicy)
+ if (!dep || !svc || !np) throw new Error('expected LlmHook resources to be written')
+ appsApi.readNamespacedDeployment.mockResolvedValue({
+ ...dep,
+ metadata: { ...dep.metadata, resourceVersion: '1' },
+ status: { readyReplicas: 1 },
+ })
+ coreApi.readNamespacedService.mockResolvedValue({
+ ...svc,
+ metadata: { ...svc.metadata, resourceVersion: '1' },
+ spec: { ...svc.spec, clusterIP: '10.0.0.1' },
+ })
+ networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ ...np,
+ metadata: { ...np.metadata, resourceVersion: '1' },
+ })
+ appsApi.replaceNamespacedDeployment.mockClear()
+ coreApi.replaceNamespacedService.mockClear()
+ networkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ return hook
+ }
+
+ it('D3: an already-synced LlmHook pass logs writes 0 and skips equal to objects', async () => {
+ const hook = await seedMatchingHook()
+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})
+ await reconciler.fullReconcile([hook])
+ const line = resyncCalls().at(-1)
+ expect(line?.[0]).toBe('Resync pass completed')
+ expect(line?.[1]).toEqual(
+ expect.objectContaining({
+ scope: 'LlmHook',
+ writes: 0,
+ objects: expect.any(Number),
+ skips: expect.any(Number),
+ passMs: expect.any(Number),
+ })
+ )
+ const fields = line?.[1] as { objects: number; skips: number }
+ expect(fields.skips).toBe(fields.objects)
+ expect(fields.objects).toBeGreaterThan(0)
+ })
+
+ it('D3: a drifted LlmHook Deployment logs writes 1', async () => {
+ const hook = await seedMatchingHook()
+ const live = await appsApi.readNamespacedDeployment()
+ appsApi.readNamespacedDeployment.mockResolvedValue({
+ ...live,
+ spec: {
+ ...live.spec,
+ template: {
+ ...live.spec?.template,
+ spec: {
+ ...live.spec?.template?.spec,
+ containers: [{ name: 'hook', image: 'registry.example.com/other:drift' }],
+ },
+ },
+ },
+ })
+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})
+ await reconciler.fullReconcile([hook])
+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ const line = resyncCalls().at(-1)
+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'LlmHook', writes: 1 }))
+ })
+
+ it('D3: a concurrent watch reconcile does not change the resync pass counters', async () => {
+ const hook = await seedMatchingHook()
+ let releaseSweep: (() => void) | undefined
+ const sweepHold = new Promise<void>(resolve => {
+ releaseSweep = resolve
+ })
+ let sweepStarted = false
+ appsApi.listNamespacedDeployment.mockImplementation(async () => {
+ sweepStarted = true
+ await sweepHold
+ return { items: [] }
+ })
+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})
+ const pass = reconciler.fullReconcile([hook])
+ await vi.waitFor(() => expect(sweepStarted).toBe(true))
+ const live = await appsApi.readNamespacedDeployment()
+ appsApi.readNamespacedDeployment.mockResolvedValue({
+ ...live,
+ spec: {
+ ...live.spec,
+ template: {
+ ...live.spec?.template,
+ spec: {
+ ...live.spec?.template?.spec,
+ containers: [{ name: 'hook', image: 'registry.example.com/watch:drift' }],
+ },
+ },
+ },
+ })
+ await reconciler.reconcile(hook)
+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)
+ releaseSweep?.()
+ await pass
+ const line = resyncCalls().at(-1)
+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'LlmHook', writes: 0 }))
+ })
+ })
})
diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.ts
index ecaa605de..8540cde7e 100644
--- a/host-context-controller/src/llmHookReconciler.ts
+++ b/host-context-controller/src/llmHookReconciler.ts
@@ -23,6 +23,7 @@
import * as k8s from '@kubernetes/client-node'
import { IntOrString } from '@kubernetes/client-node/dist/types.js'
import { createHash } from 'crypto'
+import { AsyncLocalStorage } from 'node:async_hooks'
import * as dns from 'node:dns/promises'
import { isIP } from 'node:net'
import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'
@@ -40,6 +41,7 @@ import { createsTotal } from './metrics'
import { isAllowedExternalEgressCidr, isPublicDnsHostname } from './networkPolicyReconciler'
import { HostCRD, LlmHookCRD, LlmHookCondition, LlmHookImageTarget, LlmHookStatus } from './types'
import {
+ type ResourceApplyResult,
deploymentMatchesDesired,
ensureResource,
getErrorCode,
@@ -53,6 +55,18 @@ import {
serviceMatchesDesired,
} from './utils'
+type LlmHookResyncCounters = { objects: number; writes: number; skips: number }
+
+const llmHookResyncPass = new AsyncLocalStorage<LlmHookResyncCounters>()
+
+function recordLlmHookApply(result: ResourceApplyResult): void {
+ const acc = llmHookResyncPass.getStore()
+ if (!acc) return
+ acc.objects += 1
+ if (result === 'created' || result === 'replaced') acc.writes += 1
+ else if (result === 'up_to_date') acc.skips += 1
+}
+
const GROUP = 'clerum.io'
const VERSION = 'v1alpha1'
const PLURAL_LLMHOOKS = 'llmhooks'
@@ -299,27 +313,37 @@ export class LlmHookReconciler {
* `llmhook-*` workloads whose pod key has zero live members (§3).
*/
async fullReconcile(hooks: LlmHookCRD[]): Promise<void> {
- console.log(`${LOG} Running full reconciliation for ${hooks.length} LlmHook(s)`)
- const imagePodKeys = new Set<string>()
- for (const hook of hooks) {
- const kind = classifyTarget(hook)
- if (kind === 'image') {
- const key = computePodKey(hook)
- if (key) imagePodKeys.add(key)
- } else {
- await this.reconcile(hook)
+ const started = Date.now()
+ const acc: LlmHookResyncCounters = { objects: 0, writes: 0, skips: 0 }
+ hccLogger.info('Running full reconciliation', { scope: 'LlmHook', hooks: hooks.length })
+ await llmHookResyncPass.run(acc, async () => {
+ const imagePodKeys = new Set<string>()
+ for (const hook of hooks) {
+ const kind = classifyTarget(hook)
+ if (kind === 'image') {
+ const key = computePodKey(hook)
+ if (key) imagePodKeys.add(key)
+ } else {
+ await this.reconcile(hook)
+ }
}
- }
- for (const podKey of imagePodKeys) {
- await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))
- }
- // Converge every Host's scoped egress-to-hooks policy (N1/N7) — the periodic
- // backstop for a missed watch event or an image bump that moved a pod key.
- for (const host of this.hosts.values()) {
- await this.reconcileHostEgress(host)
- }
- await this.sweepOrphanedWorkloads()
- console.log(`${LOG} Full reconciliation complete`)
+ for (const podKey of imagePodKeys) {
+ await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))
+ }
+ // Converge every Host's scoped egress-to-hooks policy (N1/N7) — the periodic
+ // backstop for a missed watch event or an image bump that moved a pod key.
+ for (const host of this.hosts.values()) {
+ await this.reconcileHostEgress(host)
+ }
+ await this.sweepOrphanedWorkloads()
+ })
+ hccLogger.info('Resync pass completed', {
+ scope: 'LlmHook',
+ objects: acc.objects,
+ writes: acc.writes,
+ skips: acc.skips,
+ passMs: Date.now() - started,
+ })
}
// ─── Members / reverse index ────────────────────────────────────────
@@ -603,7 +627,7 @@ export class LlmHookReconciler {
// workflow-recipe allowlist, but strip anything outside it at reconcile too.
const addCapabilities = (img.security?.addCapabilities ?? []).filter(cap => {
if (isWorkflowRecipeDefaultAllowedCapability(cap)) return true
- console.warn(`${LOG} pod key ${podKey}: capability "${cap}" stripped (forbidden)`)
+ hccLogger.warn('Forbidden capability stripped', { scope: LOG, podKey, capability: cap })
return false
})
@@ -759,87 +783,91 @@ export class LlmHookReconciler {
): Promise<void> {
const deployment = this.buildDeployment(podKey, members, credentialsRevision)
const name = deployment.metadata!.name!
- await ensureResource({
- read: () =>
- observeExistenceRead('Deployment', () =>
- this.appsApi.readNamespacedDeployment({
- name,
+ recordLlmHookApply(
+ await ensureResource({
+ read: () =>
+ observeExistenceRead('Deployment', () =>
+ this.appsApi.readNamespacedDeployment({
+ name,
+ namespace: config.llmHooksNamespace,
+ })
+ ),
+ create: async () => {
+ await observeCreate('Deployment', () =>
+ this.appsApi.createNamespacedDeployment({
+ namespace: config.llmHooksNamespace,
+ body: deployment,
+ })
+ )
+ hccLogger.info('Deployment created', {
+ scope: LOG,
+ deployment: name,
namespace: config.llmHooksNamespace,
})
- ),
- create: async () => {
- await observeCreate('Deployment', () =>
- this.appsApi.createNamespacedDeployment({
- namespace: config.llmHooksNamespace,
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Deployment "${name}"`,
+ logPrefix: LOG,
body: deployment,
- })
- )
- hccLogger.info('Deployment created', {
- scope: LOG,
- deployment: name,
- namespace: config.llmHooksNamespace,
- })
- },
- converge: read =>
- replaceWithConflictRetry({
- description: `Deployment "${name}"`,
- logPrefix: LOG,
- body: deployment,
- mergeExisting: preserveDeploymentAnnotations,
- isUpToDate: deploymentMatchesDesired,
- read,
- replace: body =>
- this.appsApi.replaceNamespacedDeployment({
- name,
- namespace: config.llmHooksNamespace,
- body,
- }),
- }),
- onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
- })
+ mergeExisting: preserveDeploymentAnnotations,
+ isUpToDate: deploymentMatchesDesired,
+ read,
+ replace: body =>
+ this.appsApi.replaceNamespacedDeployment({
+ name,
+ namespace: config.llmHooksNamespace,
+ body,
+ }),
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),
+ })
+ )
}
private async ensureService(podKey: string, port: number): Promise<void> {
const service = this.buildService(podKey, port)
const name = service.metadata!.name!
- await ensureResource({
- read: () =>
- observeExistenceRead('Service', () =>
- this.coreApi.readNamespacedService({
- name,
+ recordLlmHookApply(
+ await ensureResource({
+ read: () =>
+ observeExistenceRead('Service', () =>
+ this.coreApi.readNamespacedService({
+ name,
+ namespace: config.llmHooksNamespace,
+ })
+ ),
+ create: async () => {
+ await observeCreate('Service', () =>
+ this.coreApi.createNamespacedService({
+ namespace: config.llmHooksNamespace,
+ body: service,
+ })
+ )
+ hccLogger.info('Service created', {
+ scope: LOG,
+ service: name,
namespace: config.llmHooksNamespace,
})
- ),
- create: async () => {
- await observeCreate('Service', () =>
- this.coreApi.createNamespacedService({
- namespace: config.llmHooksNamespace,
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `Service "${name}"`,
+ logPrefix: LOG,
body: service,
- })
- )
- hccLogger.info('Service created', {
- scope: LOG,
- service: name,
- namespace: config.llmHooksNamespace,
- })
- },
- converge: read =>
- replaceWithConflictRetry({
- description: `Service "${name}"`,
- logPrefix: LOG,
- body: service,
- mergeExisting: preserveServiceAssignedFields,
- isUpToDate: serviceMatchesDesired,
- read,
- replace: body =>
- this.coreApi.replaceNamespacedService({
- name,
- namespace: config.llmHooksNamespace,
- body,
- }),
- }),
- onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
- })
+ mergeExisting: preserveServiceAssignedFields,
+ isUpToDate: serviceMatchesDesired,
+ read,
+ replace: body =>
+ this.coreApi.replaceNamespacedService({
+ name,
+ namespace: config.llmHooksNamespace,
+ body,
+ }),
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),
+ })
+ )
}
private async ensureNetworkPolicy(
@@ -854,30 +882,32 @@ export class LlmHookReconciler {
private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {
const name = policy.metadata!.name!
const namespace = policy.metadata!.namespace ?? config.llmHooksNamespace
- await ensureResource({
- read: () =>
- observeExistenceRead('NetworkPolicy', () =>
- this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
- ),
- create: async () => {
- await observeCreate('NetworkPolicy', () =>
- this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
- )
- hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })
- },
- converge: read =>
- replaceWithConflictRetry({
- description: `NetworkPolicy "${name}"`,
- logPrefix: LOG,
- body: policy,
- mergeExisting: preserveObjectAnnotations,
- isUpToDate: networkPolicyMatchesDesired,
- read,
- replace: body =>
- this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
- }),
- onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
- })
+ recordLlmHookApply(
+ await ensureResource({
+ read: () =>
+ observeExistenceRead('NetworkPolicy', () =>
+ this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
+ ),
+ create: async () => {
+ await observeCreate('NetworkPolicy', () =>
+ this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })
+ )
+ hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })
+ },
+ converge: read =>
+ replaceWithConflictRetry({
+ description: `NetworkPolicy "${name}"`,
+ logPrefix: LOG,
+ body: policy,
+ mergeExisting: preserveObjectAnnotations,
+ isUpToDate: networkPolicyMatchesDesired,
+ read,
+ replace: body =>
+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),
+ }),
+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),
+ })
+ )
}
/**
@@ -894,9 +924,12 @@ export class LlmHookReconciler {
const svc = hook.spec.target?.service
if (!svc?.name || !svc.namespace || !svc.port) return
if (svc.namespace !== config.llmHooksNamespace) {
- console.warn(
- `${LOG} service-target hook "${hook.name}" → Service ${svc.namespace}/${svc.name} outside ${config.llmHooksNamespace}; ingress not enforced by HCC`
- )
+ hccLogger.warn('Service-target hook points outside llm-hooks; ingress not enforced', {
+ scope: LOG,
+ hook: hook.name,
+ serviceNamespace: svc.namespace,
+ service: svc.name,
+ })
return
}
@@ -905,9 +938,12 @@ export class LlmHookReconciler {
// fail-closed under the namespace default-deny instead.
const selector = await this.readServiceSelector(svc.name, svc.namespace)
if (!selector) {
- console.warn(
- `${LOG} service-target hook "${hook.name}": Service ${svc.namespace}/${svc.name} missing or selector-less; leaving fail-closed under default-deny`
- )
+ hccLogger.warn('Service-target hook missing selector; leaving fail-closed', {
+ scope: LOG,
+ hook: hook.name,
+ serviceNamespace: svc.namespace,
+ service: svc.name,
+ })
return
}
@@ -958,9 +994,12 @@ export class LlmHookReconciler {
const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {
await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
- console.log(`${LOG} Deleted NetworkPolicy "${name}"`)
+ hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name })
} else {
- console.warn(`${LOG} Skipping NetworkPolicy "${name}" delete — not HCC-owned`)
+ hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {
+ scope: LOG,
+ policy: name,
+ })
}
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -1085,7 +1124,7 @@ export class LlmHookReconciler {
const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {
await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
- console.log(`${LOG} Deleted NetworkPolicy "${name}" (${ns})`)
+ hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name, namespace: ns })
}
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -1110,9 +1149,12 @@ export class LlmHookReconciler {
const dep = await this.appsApi.readNamespacedDeployment({ name, namespace: ns })
if (this.isHccOwned(dep, podKey)) {
await this.appsApi.deleteNamespacedDeployment({ name, namespace: ns })
- console.log(`${LOG} Deleted Deployment "${name}" (0 members)`)
+ hccLogger.info('Deployment deleted (0 members)', { scope: LOG, deployment: name })
} else {
- console.warn(`${LOG} Skipping Deployment "${name}" delete — not HCC-owned`)
+ hccLogger.warn('Skipping Deployment delete — not HCC-owned', {
+ scope: LOG,
+ deployment: name,
+ })
}
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -1122,9 +1164,9 @@ export class LlmHookReconciler {
const svc = await this.coreApi.readNamespacedService({ name, namespace: ns })
if (this.isHccOwned(svc, podKey)) {
await this.coreApi.deleteNamespacedService({ name, namespace: ns })
- console.log(`${LOG} Deleted Service "${name}" (0 members)`)
+ hccLogger.info('Service deleted (0 members)', { scope: LOG, service: name })
} else {
- console.warn(`${LOG} Skipping Service "${name}" delete — not HCC-owned`)
+ hccLogger.warn('Skipping Service delete — not HCC-owned', { scope: LOG, service: name })
}
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -1134,9 +1176,12 @@ export class LlmHookReconciler {
const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })
if (this.isHccOwned(np, podKey)) {
await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })
- console.log(`${LOG} Deleted NetworkPolicy "${name}" (0 members)`)
+ hccLogger.info('NetworkPolicy deleted (0 members)', { scope: LOG, policy: name })
} else {
- console.warn(`${LOG} Skipping NetworkPolicy "${name}" delete — not HCC-owned`)
+ hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {
+ scope: LOG,
+ policy: name,
+ })
}
} catch (error) {
if (getErrorCode(error) !== 404) throw error
@@ -1158,14 +1203,14 @@ export class LlmHookReconciler {
})
deployments = resp.items ?? []
} catch (error) {
- console.error(`${LOG} Orphan sweep: failed to list Deployments:`, error)
+ hccLogger.error('Orphan sweep failed to list Deployments', { scope: LOG, err: error })
return
}
for (const dep of deployments) {
const podKey = dep.metadata?.labels?.[HOOK_PODKEY_LABEL]
if (!podKey) continue
if (this.membersForPodKey(podKey).length > 0) continue
- console.log(`${LOG} Orphan sweep: pod key ${podKey} has 0 members — deleting workload`)
+ hccLogger.info('Orphan sweep deleting workload with 0 members', { scope: LOG, podKey })
await this.runSerialized(podKey, () => this.gcPodKey(podKey))
}
}
@@ -1211,7 +1256,7 @@ export class LlmHookReconciler {
}
}
} catch (error) {
- console.warn(`${LOG} Failed to read observedDigest for pod key ${podKey}:`, error)
+ hccLogger.warn('Failed to read observedDigest', { scope: LOG, podKey, err: error })
}
return ''
}
@@ -1243,7 +1288,7 @@ export class LlmHookReconciler {
})) as { metadata?: { resourceVersion?: string }; status?: LlmHookStatus }
} catch (error) {
if (getErrorCode(error) === 404) return
- console.warn(`${LOG} Failed to read status for "${hook.name}":`, error)
+ hccLogger.warn('Failed to read LlmHook status', { scope: LOG, hook: hook.name, err: error })
return
}
@@ -1318,7 +1363,11 @@ export class LlmHookReconciler {
if (getErrorCode(error) === 404) return
const code = getErrorCode(error)
if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) continue
- console.warn(`${LOG} Failed to write status on "${hook.name}":`, error)
+ hccLogger.warn('Failed to write LlmHook status', {
+ scope: LOG,
+ hook: hook.name,
+ err: error,
+ })
return
}
}
diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.ts
index 2b1989900..7fcb2aacb 100644
--- a/host-context-controller/src/utils.ts
+++ b/host-context-controller/src/utils.ts
@@ -27,6 +27,9 @@ export async function observeCreate<T>(kind: CreateKind, create: () => Promise<T
}
}
+/** Outcome of a read-first create-or-replace. Callers may ignore this value. */
+export type ResourceApplyResult = 'created' | 'replaced' | 'up_to_date' | 'missing' | 'not_allowed'
+
/** Observe one existence GET without changing its value or error handling. */
export async function observeExistenceRead<T>(
kind: CreateKind,
@@ -53,12 +56,12 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>
create: () => Promise<unknown>
existing?: T | null
/** False preserves a caller's non-throwing failure without counting a skip. */
- converge: (read: () => Promise<T>, observed?: T) => Promise<void | false>
+ converge: (read: () => Promise<T>, observed?: T) => Promise<void | false | ResourceApplyResult>
onSkipped: () => void
mutationAllowed?: () => boolean
-}): Promise<void> {
+}): Promise<ResourceApplyResult> {
const { read, create, converge, onSkipped, mutationAllowed } = opts
- if (mutationAllowed && !mutationAllowed()) return
+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
const validate = (value: T): T => {
if (!value || typeof value.metadata?.name !== 'string' || !value.metadata.name.trim()) {
@@ -92,7 +95,7 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>
} else if (existing !== null) {
validate(existing)
}
- if (mutationAllowed && !mutationAllowed()) return
+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
if (existing !== null) {
let snapshot: T | undefined = existing
@@ -108,16 +111,22 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>
if (result !== false && latestReadSucceeded && (!mutationAllowed || mutationAllowed())) {
onSkipped()
}
- return
+ if (result === false) return 'not_allowed'
+ if (typeof result === 'string') return result
+ return latestReadSucceeded ? 'up_to_date' : 'missing'
}
try {
await create()
+ return 'created'
} catch (error) {
if (error == null || getErrorCode(error) !== 409) throw error
- if (mutationAllowed && !mutationAllowed()) return
+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
// Absence observed before the POST cannot survive a create conflict.
- await converge(readFresh)
+ const result = await converge(readFresh)
+ if (result === false) return 'not_allowed'
+ if (typeof result === 'string') return result
+ return latestReadSucceeded ? 'up_to_date' : 'missing'
}
}
@@ -161,7 +170,7 @@ export async function replaceWithConflictRetry<
/** Rebuild desired state before each attempt without an unused eager body. */
| { body?: never; resolveBody: () => T | Promise<T> }
)
-): Promise<void> {
+): Promise<ResourceApplyResult> {
const {
description,
logPrefix,
@@ -185,7 +194,7 @@ export async function replaceWithConflictRetry<
scope: logPrefix,
description,
})
- return
+ return 'missing'
}
throw err
}
@@ -198,16 +207,16 @@ export async function replaceWithConflictRetry<
}
const next = mergeExisting ? mergeExisting(base, existing) : base
if (isUpToDate?.(next, existing)) {
- if (mutationAllowed && !mutationAllowed()) return
+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
writeSkipsTotal.inc({ kind: next.kind ?? 'unknown' })
- return
+ return 'up_to_date'
}
- if (mutationAllowed && !mutationAllowed()) return
+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'
try {
await replace(next)
writesTotal.inc({ kind: next.kind ?? 'unknown' })
hccLogger.info('Kubernetes resource updated', { scope: logPrefix, description, attempt })
- return
+ return 'replaced'
} catch (err) {
if (getErrorCode(err) === 409 && attempt < maxAttempts) {
// Jittered backoff: 25–125ms on attempt 1, 50–250ms on attempt 2.
@@ -219,6 +228,7 @@ export async function replaceWithConflictRetry<
throw err
}
}
+ throw new Error(`replaceWithConflictRetry exhausted attempts for ${description}`)
}
function mergeAnnotations(
@@ -605,8 +615,8 @@ export async function applyNetworkPolicy(
validateExisting?: (existing: k8s.V1NetworkPolicy) => void,
missingIsError = false,
existing?: k8s.V1NetworkPolicy | null
-): Promise<void> {
- await ensureResource<k8s.V1NetworkPolicy>({
+): Promise<ResourceApplyResult> {
+ return ensureResource<k8s.V1NetworkPolicy>({
existing,
mutationAllowed,
read: () =>