Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(hcc): skip no-op gfsc-reader Deployment replaces
Alfredo Lopezd50f97532909
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex ac10a721e..85c4ed718 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.337","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.337","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/codex-catalog-projection": "file:../packages/codex-catalog-projection",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 921d081c5..1c0ad4de2 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.336",+ "version": "0.1.337","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.tsindex 19cbc2573..7524b664d 100644--- a/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts+++ b/host-context-controller/src/__tests__/applyNetworkPolicyNoopGate.test.ts@@ -123,7 +123,7 @@ describe('applyNetworkPolicy no-op gate', () => {await expect(applyNetworkPolicy(api as unknown as k8s.NetworkingV1Api, 'np', 'ns', desiredPolicy())- ).resolves.toBeUndefined()+ ).resolves.toBe('missing')expect(api.createNamespacedNetworkPolicy).toHaveBeenCalledOnce()expect(api.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)@@ -199,7 +199,7 @@ describe('applyNetworkPolicy no-op gate', () => {'[NetPol]',mutationAllowed)- ).resolves.toBeUndefined()+ ).resolves.toBe('not_allowed')expect(mutationAllowed).toHaveBeenCalledOnce()expect(api.readNamespacedNetworkPolicy).not.toHaveBeenCalled()diff --git a/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts b/host-context-controller/src/__tests__/deploymentWriterInventory.test.tsindex b0242b642..d2fbc1d4c 100644--- a/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts+++ b/host-context-controller/src/__tests__/deploymentWriterInventory.test.ts@@ -9,9 +9,10 @@ const GATED = new Set(['reconciler.ts','sharedFileSystemReconciler.ts','llmHookReconciler.ts',+ 'k8s/gfsK8sApi.ts',])-const EXEMPT = new Set(['k8s/gfsK8sApi.ts', 'statelessLifecycleExecutor.ts'])+const EXEMPT = new Set(['statelessLifecycleExecutor.ts'])/** Any call, including `replace(params)` — not only inline `({ ... })`. */const CALL = /replaceNamespacedDeployment\s*\(/g@@ -84,6 +85,11 @@ function isDeploymentGated(block: string): boolean {)}+/** Replica-only scale in K8sGfsApi is not a template replace; it is not a no-op gate. */+function isReplicaScaleReplace(block: string): boolean {+ return /replicas/.test(block) && /mergeExisting/.test(block) && !isDeploymentGated(block)+}+describe('Deployment writer inventory', () => {it('every replaceNamespacedDeployment call is gated or explicitly exempt', () => {const hits: Array<{ rel: string; count: number; gated: boolean }> = []@@ -119,6 +125,7 @@ describe('Deployment writer inventory', () => {block,`${rel} call at index ${index} is not inside replaceWithConflictRetry`).not.toBeNull()+ if (isReplicaScaleReplace(block ?? '')) continueexpect(isDeploymentGated(block ?? ''),`${rel} call at index ${index} is missing an isUpToDate deploymentMatchesDesired gate`diff --git a/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.json b/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.jsonnew file mode 100644index 000000000..95d1dd6f3--- /dev/null+++ b/host-context-controller/src/__tests__/fixtures/629/gfsc-reader.deployment.json@@ -0,0 +1,281 @@+{+ "apiVersion": "apps/v1",+ "kind": "Deployment",+ "metadata": {+ "annotations": {+ "deployment.kubernetes.io/revision": "511",+ "clerum.io/gfsc-template-hash": "504c150f07082276"+ },+ "labels": {+ "clerum.io/globalfilesystem": "gfs",+ "clerum.io/managed-by": "host-context-controller"+ },+ "name": "gfsc-reader",+ "namespace": "gfs"+ },+ "spec": {+ "progressDeadlineSeconds": 600,+ "replicas": 2,+ "revisionHistoryLimit": 10,+ "selector": {+ "matchLabels": {+ "app": "gfs-controller",+ "clerum.io/gfsc-role": "reader"+ }+ },+ "strategy": {+ "rollingUpdate": {+ "maxSurge": 1,+ "maxUnavailable": 0+ },+ "type": "RollingUpdate"+ },+ "template": {+ "metadata": {+ "annotations": {+ "clerum.io/gfsc-template-hash": "504c150f07082276"+ },+ "labels": {+ "app": "gfs-controller",+ "clerum.io/gfsc-role": "reader"+ }+ },+ "spec": {+ "affinity": {+ "podAffinity": {+ "requiredDuringSchedulingIgnoredDuringExecution": [+ {+ "labelSelector": {+ "matchLabels": {+ "app": "gfs-controller",+ "clerum.io/gfsc-role": "writer"+ }+ },+ "topologyKey": "kubernetes.io/hostname"+ }+ ]+ }+ },+ "automountServiceAccountToken": false,+ "containers": [+ {+ "env": [+ {+ "name": "GFS_PORT",+ "value": "8087"+ },+ {+ "name": "GFS_STORAGE_PATH",+ "value": "/data/gfs"+ },+ {+ "name": "GFS_STORAGE_ROLE",+ "value": "reader"+ },+ {+ "name": "GFS_DRIVE_NAME",+ "value": "main"+ },+ {+ "name": "GFS_TOKEN_AUDIENCE",+ "value": "gfs-controller"+ },+ {+ "name": "GFS_JWT_PUBLIC_KEY",+ "valueFrom": {+ "configMapKeyRef": {+ "key": "jwt-public-key",+ "name": "synthetic-gfs-config"+ }+ }+ },+ {+ "name": "GFS_PG_CONNECTION_STRING",+ "valueFrom": {+ "secretKeyRef": {+ "key": "connection-string",+ "name": "synthetic-reader-db"+ }+ }+ },+ {+ "name": "GFS_MAX_WRITE_BODY_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_PROTOCOL_MAX_FILE_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_PRODUCT_MAX_FILE_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_FILE_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_PREFERRED_CHUNK_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_CHUNK_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MIN_PART_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_PART_COUNT",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_SESSION_TTL_MS",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_COMPLETED_RECEIPT_TTL_MS",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_STALE_PART_LEASE_MS",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_ACTIVE_PER_SUBJECT",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_ACTIVE_GLOBAL",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_CONCURRENT_PARTS_PER_SESSION",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_CONCURRENT_PART_STREAMS_GLOBAL",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_INSTABILITY_FAILURE_THRESHOLD",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MAX_CONCURRENT_FINALIZATIONS",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_MIN_FREE_BYTES",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_PART_TIMEOUT_MS",+ "value": "1"+ },+ {+ "name": "GFS_UPLOAD_FINALIZE_TIMEOUT_MS",+ "value": "1"+ }+ ],+ "image": "registry.example.com/gfs-controller:synthetic",+ "imagePullPolicy": "IfNotPresent",+ "livenessProbe": {+ "failureThreshold": 3,+ "httpGet": {+ "path": "/healthz",+ "port": "http",+ "scheme": "HTTP"+ },+ "initialDelaySeconds": 10,+ "periodSeconds": 30,+ "successThreshold": 1,+ "timeoutSeconds": 2+ },+ "name": "gfsc",+ "ports": [+ {+ "containerPort": 8087,+ "name": "http",+ "protocol": "TCP"+ }+ ],+ "readinessProbe": {+ "failureThreshold": 3,+ "httpGet": {+ "path": "/readyz",+ "port": "http",+ "scheme": "HTTP"+ },+ "initialDelaySeconds": 2,+ "periodSeconds": 5,+ "successThreshold": 1,+ "timeoutSeconds": 2+ },+ "resources": {+ "limits": {+ "cpu": "500m",+ "memory": "512Mi"+ },+ "requests": {+ "cpu": "100m",+ "memory": "128Mi"+ }+ },+ "securityContext": {+ "allowPrivilegeEscalation": false,+ "capabilities": {+ "drop": ["ALL"]+ },+ "readOnlyRootFilesystem": true+ },+ "startupProbe": {+ "failureThreshold": 30,+ "httpGet": {+ "path": "/readyz",+ "port": "http",+ "scheme": "HTTP"+ },+ "initialDelaySeconds": 2,+ "periodSeconds": 2,+ "successThreshold": 1,+ "timeoutSeconds": 1+ },+ "terminationMessagePath": "/dev/termination-log",+ "terminationMessagePolicy": "File",+ "volumeMounts": [+ {+ "mountPath": "/data/gfs",+ "name": "drive",+ "readOnly": true+ }+ ]+ }+ ],+ "dnsPolicy": "ClusterFirst",+ "restartPolicy": "Always",+ "schedulerName": "default-scheduler",+ "securityContext": {+ "fsGroup": 1000,+ "fsGroupChangePolicy": "OnRootMismatch",+ "runAsGroup": 1000,+ "runAsNonRoot": true,+ "runAsUser": 1000,+ "seccompProfile": {+ "type": "RuntimeDefault"+ }+ },+ "terminationGracePeriodSeconds": 30,+ "volumes": [+ {+ "name": "drive",+ "persistentVolumeClaim": {+ "claimName": "gfs-drive"+ }+ }+ ]+ }+ }+ }+}diff --git a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.tsindex b7688dc2b..5ddd0eec1 100644--- a/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts+++ b/host-context-controller/src/__tests__/replaceWithConflictRetryOrder.test.ts@@ -171,7 +171,7 @@ describe('replaceWithConflictRetry order and retry', () => {read,replace,})- ).resolves.toBeUndefined()+ ).resolves.toBe('missing')expect(read).toHaveBeenCalledTimes(2)expect(validateExisting).toHaveBeenCalledOnce()diff --git a/host-context-controller/src/gfsReconciler.test.ts b/host-context-controller/src/gfsReconciler.test.tsindex d2fd7965b..6cb69d8f0 100644--- a/host-context-controller/src/gfsReconciler.test.ts+++ b/host-context-controller/src/gfsReconciler.test.ts@@ -1,8 +1,11 @@-import { describe, expect, it } from 'vitest'+import { beforeEach, describe, expect, it, vi } from 'vitest'import type * as k8s from '@kubernetes/client-node'import { GfsK8sApi, GfsReconciler, GfsSeedClient } from './gfsReconciler'import { GfsFactoryConfig } from './k8s/gfsFactory'+import { K8sGfsApi } from './k8s/gfsK8sApi'+import { hccLogger } from './logger'import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'+import type { ResourceApplyResult } from './utils'const config: GfsFactoryConfig = {gfsNamespace: 'gfs',@@ -48,38 +51,51 @@ class FakeApi implements GfsK8sApi {writerAvailabilitySequence: boolean[] = []failPvc = falsestatusFailuresRemaining = 0+ applyOutcome: 'created' | 'replaced' | 'up_to_date' = 'replaced'+ readerApplyOutcome?: 'created' | 'replaced' | 'up_to_date'- async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<void> {+ async applyPvc(pvc: k8s.V1PersistentVolumeClaim): Promise<ResourceApplyResult> {if (this.failPvc) throw new Error('simulated PVC apply failure')this.pvcs.push(pvc.metadata?.name ?? '')this.operations.push(`pvc/${pvc.metadata?.name ?? ''}`)+ return this.applyOutcome}async deploymentNeedsUpdate(dep: k8s.V1Deployment): Promise<boolean> {return dep.metadata?.name === 'gfsc-writer' ? this.writerNeedsUpdate : false}- async scaleDeployment(name: string, _namespace: string, replicas: number): Promise<void> {+ async scaleDeployment(+ name: string,+ _namespace: string,+ replicas: number+ ): Promise<ResourceApplyResult> {this.operations.push(`scale/${name}/${replicas}`)+ return this.applyOutcome === 'up_to_date' ? 'up_to_date' : 'replaced'}- async applyDeployment(dep: k8s.V1Deployment): Promise<void> {+ async applyDeployment(dep: k8s.V1Deployment): Promise<ResourceApplyResult> {const name = dep.metadata?.name ?? ''this.deployments.push(name)this.deploymentManifests.push(dep)this.operations.push(`deploy/${name}`)+ if (name === 'gfsc-reader' && this.readerApplyOutcome) return this.readerApplyOutcome+ return this.applyOutcome}- async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget): Promise<void> {+ async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget): Promise<ResourceApplyResult> {const name = pdb.metadata?.name ?? ''this.pdbs.push(name)this.operations.push(`pdb/${name}`)+ return this.applyOutcome}- async applyService(svc: k8s.V1Service): Promise<void> {+ async applyService(svc: k8s.V1Service): Promise<ResourceApplyResult> {const name = svc.metadata?.name ?? ''this.services.push(name)this.operations.push(`svc/${name}`)+ return this.applyOutcome}- async applyNetworkPolicy(np: k8s.V1NetworkPolicy): Promise<void> {+ async applyNetworkPolicy(np: k8s.V1NetworkPolicy): Promise<ResourceApplyResult> {const name = np.metadata?.name ?? ''this.netpols.push(name)this.operations.push(`np/${name}`)+ return this.applyOutcome}async isDeploymentAvailable(name: string): Promise<boolean> {if (name === 'gfsc-writer' && this.writerAvailabilitySequence.length > 0) {@@ -266,7 +282,9 @@ describe('GfsReconciler.reconcile', () => {const reconciler = new GfsReconciler(api, config)await expect(reconciler.reconcile(gfs)).rejects.toThrow('simulated status patch failure')- await expect(reconciler.reconcile(gfs)).resolves.toBeUndefined()+ await expect(reconciler.reconcile(gfs)).resolves.toEqual(+ expect.objectContaining({ objects: expect.any(Number) })+ )expect(api.statuses).toHaveLength(1)expect(api.statuses[0]?.phase).toBe('Ready')@@ -296,6 +314,164 @@ describe('GfsReconciler.fullReconcile', () => {})})+function createStoreBackedGfsApi() {+ const store = new Map<string, Record<string, unknown>>()+ const reads = { total: 0, secondPass: 0 }+ const replaces = { deployment: 0, pdb: 0, secondPassDeployment: 0, secondPassPdb: 0 }+ let pass = 1++ const keyOf = (kind: string, name: string) => `${kind}/${name}`++ const read =+ (kind: string) =>+ async ({ name }: { name: string }) => {+ reads.total += 1+ if (pass === 2) reads.secondPass += 1+ const stored = store.get(keyOf(kind, name))+ if (!stored) throw { code: 404 }+ return structuredClone(stored)+ }++ const create =+ (kind: string) =>+ async ({ body }: { body: { metadata?: { name?: string } } }) => {+ const name = body.metadata?.name ?? ''+ const stored = structuredClone(body) as Record<string, unknown>+ const metadata = {+ ...((stored.metadata as Record<string, unknown> | undefined) ?? {}),+ resourceVersion: '1',+ ...(kind === 'Deployment'+ ? {+ generation: 1,+ annotations: {+ ...(((stored.metadata as { annotations?: Record<string, string> } | undefined)+ ?.annotations ?? {}) as Record<string, string>),+ 'deployment.kubernetes.io/revision': '1',+ },+ }+ : {}),+ }+ stored.metadata = metadata+ if (kind === 'Deployment') {+ stored.status = { availableReplicas: 1, observedGeneration: 1 }+ }+ store.set(keyOf(kind, name), stored)+ return structuredClone(stored)+ }++ const replace =+ (kind: string) =>+ async ({ body }: { body: { metadata?: { name?: string } } }) => {+ if (kind === 'Deployment') {+ replaces.deployment += 1+ if (pass === 2) replaces.secondPassDeployment += 1+ }+ if (kind === 'PodDisruptionBudget') {+ replaces.pdb += 1+ if (pass === 2) replaces.secondPassPdb += 1+ }+ const name = body.metadata?.name ?? ''+ const stored = structuredClone(body) as Record<string, unknown>+ stored.metadata = {+ ...((stored.metadata as Record<string, unknown> | undefined) ?? {}),+ resourceVersion: '2',+ }+ store.set(keyOf(kind, name), stored)+ return structuredClone(stored)+ }++ const sdk = {+ readNamespacedPersistentVolumeClaim: read('PersistentVolumeClaim'),+ createNamespacedPersistentVolumeClaim: create('PersistentVolumeClaim'),+ readNamespacedService: read('Service'),+ createNamespacedService: create('Service'),+ readNamespacedDeployment: read('Deployment'),+ createNamespacedDeployment: create('Deployment'),+ replaceNamespacedDeployment: replace('Deployment'),+ readNamespacedPodDisruptionBudget: read('PodDisruptionBudget'),+ createNamespacedPodDisruptionBudget: create('PodDisruptionBudget'),+ replaceNamespacedPodDisruptionBudget: replace('PodDisruptionBudget'),+ readNamespacedNetworkPolicy: read('NetworkPolicy'),+ createNamespacedNetworkPolicy: create('NetworkPolicy'),+ replaceNamespacedNetworkPolicy: replace('NetworkPolicy'),+ patchNamespacedCustomObjectStatus: async () => ({}),+ }++ return {+ api: new K8sGfsApi(+ sdk as unknown as k8s.CoreV1Api,+ sdk as unknown as k8s.AppsV1Api,+ sdk as unknown as k8s.NetworkingV1Api,+ sdk as unknown as k8s.PolicyV1Api,+ sdk as unknown as k8s.CustomObjectsApi+ ),+ reads,+ replaces,+ beginSecondPass() {+ pass = 2+ },+ }+}++describe('GfsReconciler Deployment gate across two reconciles (T7)', () => {+ it('T7: second reconcile does not write Deployment or PDB and still reads', async () => {+ const cluster = createStoreBackedGfsApi()+ const reconciler = new GfsReconciler(cluster.api, config, undefined, 0, 0)+ await reconciler.reconcile(gfs)+ cluster.beginSecondPass()+ await reconciler.reconcile({ ...gfs, status: { phase: 'Ready' } })+ expect(cluster.reads.secondPass).toBeGreaterThan(0)+ expect(cluster.replaces.secondPassDeployment).toBe(0)+ expect(cluster.replaces.secondPassPdb).toBe(0)+ })+})++describe('GfsReconciler resync pass log (D3)', () => {+ beforeEach(() => {+ vi.restoreAllMocks()+ })++ function resyncCalls() {+ return vi+ .mocked(hccLogger.info)+ .mock.calls.filter(([message]) => message === 'Resync pass completed')+ }++ it('D3: an already-synced GFS pass logs writes 0 and skips equal to objects', async () => {+ const api = new FakeApi()+ api.writerNeedsUpdate = false+ api.applyOutcome = 'up_to_date'+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})+ await new GfsReconciler(api, config).fullReconcile([gfs])+ expect(info).toHaveBeenCalled()+ const line = resyncCalls().at(-1)+ expect(line?.[0]).toBe('Resync pass completed')+ expect(line?.[1]).toEqual(+ expect.objectContaining({+ scope: 'GFS',+ writes: 0,+ objects: expect.any(Number),+ skips: expect.any(Number),+ passMs: expect.any(Number),+ })+ )+ const fields = line?.[1] as { objects: number; skips: number }+ expect(fields.skips).toBe(fields.objects)+ expect(fields.objects).toBeGreaterThan(0)+ })++ it('D3: a drifted reader logs writes 1', async () => {+ const api = new FakeApi()+ api.writerNeedsUpdate = false+ api.applyOutcome = 'up_to_date'+ api.readerApplyOutcome = 'replaced'+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})+ await new GfsReconciler(api, config).fullReconcile([gfs])+ const line = resyncCalls().at(-1)+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'GFS', writes: 1 }))+ })+})+describe('GfsReconciler.reconcileDelete', () => {it('retains the PVC by default (tears down broker, keeps bytes)', async () => {const api = new FakeApi()diff --git a/host-context-controller/src/gfsReconciler.ts b/host-context-controller/src/gfsReconciler.tsindex 0462a12e6..b6adece9a 100644--- a/host-context-controller/src/gfsReconciler.ts+++ b/host-context-controller/src/gfsReconciler.ts@@ -36,20 +36,44 @@ import {writerDeploymentName,writerServiceName,} from './k8s/gfsFactory'+import { hccLogger } from './logger'import type { GlobalFileSystemCRD, GlobalFileSystemStatus } from './types'+import type { ResourceApplyResult } from './utils'/*** Minimal Kubernetes surface the reconciler needs. PVCs are create-if-absent* (their spec is immutable post-bind); everything else is create-or-replace.*/+export interface GfsResyncStats {+ objects: number+ writes: number+ skips: number+}++function emptyGfsResyncStats(): GfsResyncStats {+ return { objects: 0, writes: 0, skips: 0 }+}++function recordGfsApply(+ stats: GfsResyncStats,+ result: ResourceApplyResult | 'writer_hash_skip'+): void {+ stats.objects += 1+ if (result === 'created' || result === 'replaced') stats.writes += 1+ else if (result === 'up_to_date' || result === 'writer_hash_skip') stats.skips += 1+}+export interface GfsK8sApi {- applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<void>+ applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<ResourceApplyResult>deploymentNeedsUpdate(dep: k8s.V1Deployment, namespace: string): Promise<boolean>- scaleDeployment(name: string, namespace: string, replicas: number): Promise<void>- applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<void>- applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget, namespace: string): Promise<void>- applyService(svc: k8s.V1Service, namespace: string): Promise<void>- applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<void>+ scaleDeployment(name: string, namespace: string, replicas: number): Promise<ResourceApplyResult>+ applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<ResourceApplyResult>+ applyPodDisruptionBudget(+ pdb: k8s.V1PodDisruptionBudget,+ namespace: string+ ): Promise<ResourceApplyResult>+ applyService(svc: k8s.V1Service, namespace: string): Promise<ResourceApplyResult>+ applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<ResourceApplyResult>/** True when the named Deployment reports at least one available replica. */isDeploymentAvailable(name: string, namespace: string): Promise<boolean>deleteDeployment(name: string, namespace: string): Promise<void>@@ -116,29 +140,41 @@ export class GfsReconciler {await this.api.patchStatus(gfs.name, gfs.namespace, status)}- async reconcile(gfs: GlobalFileSystemCRD): Promise<void> {+ async reconcile(gfs: GlobalFileSystemCRD): Promise<GfsResyncStats> {+ const stats = emptyGfsResyncStats()const ns = this.config.gfsNamespaceconst writerDeployment = buildDeployment(gfs, this.config, 'writer')const readerDeployment = buildDeployment(gfs, this.config, 'reader')const writerName = writerDeploymentName(gfs)const readerName = readerDeploymentName(gfs)- await this.api.applyPvc(buildPvc(gfs, this.config), ns)+ recordGfsApply(stats, await this.api.applyPvc(buildPvc(gfs, this.config), ns))const writerNeedsUpdate = await this.api.deploymentNeedsUpdate(writerDeployment, ns)if (writerNeedsUpdate) {// With standard-rwo, reader pods keep RO mounts on the writer's node.// During a writer template rollout, scale them down before applying the// writer so Kubernetes can attach the PVC to the writer's next node.- await this.api.scaleDeployment(readerName, ns, 0)+ recordGfsApply(stats, await this.api.scaleDeployment(readerName, ns, 0))}- await this.api.applyPodDisruptionBudget(buildPodDisruptionBudget(gfs, this.config), ns)- await this.api.applyService(buildService(gfs, this.config), ns)- await this.api.applyService(buildWriterService(gfs, this.config), ns)- await this.api.applyNetworkPolicy(buildIngressNetworkPolicy(gfs, this.config), ns)- await this.api.applyNetworkPolicy(buildEgressNetworkPolicy(gfs, this.config), ns)+ recordGfsApply(+ stats,+ await this.api.applyPodDisruptionBudget(buildPodDisruptionBudget(gfs, this.config), ns)+ )+ recordGfsApply(stats, await this.api.applyService(buildService(gfs, this.config), ns))+ recordGfsApply(stats, await this.api.applyService(buildWriterService(gfs, this.config), ns))+ recordGfsApply(+ stats,+ await this.api.applyNetworkPolicy(buildIngressNetworkPolicy(gfs, this.config), ns)+ )+ recordGfsApply(+ stats,+ await this.api.applyNetworkPolicy(buildEgressNetworkPolicy(gfs, this.config), ns)+ )if (writerNeedsUpdate) {- await this.api.applyDeployment(writerDeployment, ns)+ recordGfsApply(stats, await this.api.applyDeployment(writerDeployment, ns))+ } else {+ recordGfsApply(stats, 'writer_hash_skip')}let writerReady = writerNeedsUpdate@@ -148,11 +184,11 @@ export class GfsReconciler {// A current writer template can still be unavailable after a node drain or// pod eviction. Readers may retain RO mounts on the old node; release them// so the single RWO writer can attach first, then restore readers below.- await this.api.scaleDeployment(readerName, ns, 0)+ recordGfsApply(stats, await this.api.scaleDeployment(readerName, ns, 0))writerReady = await this.waitForWriterAvailable(gfs, ns)}if (writerReady) {- await this.api.applyDeployment(readerDeployment, ns)+ recordGfsApply(stats, await this.api.applyDeployment(readerDeployment, ns))}const phase = writerReady ? 'Ready' : 'Initializing'@@ -168,6 +204,7 @@ export class GfsReconciler {await this.seedClient.seedRootDirectories(gfs)this.seeded.add(gfsKey(gfs))}+ return stats}/**@@ -177,17 +214,29 @@ export class GfsReconciler {* not a swallowed error: the failure is surfaced and the loop is idempotent.*/async fullReconcile(gfses: GlobalFileSystemCRD[]): Promise<void> {+ const started = Date.now()+ const totals = emptyGfsResyncStats()for (const gfs of gfses) {try {- await this.reconcile(gfs)+ const pass = await this.reconcile(gfs)+ totals.objects += pass.objects+ totals.writes += pass.writes+ totals.skips += pass.skips} catch (err) {- console.error(- `[gfsReconciler] reconcile failed for "${gfsKey(gfs)}": ${- err instanceof Error ? err.message : String(err)- }`- )+ hccLogger.error('reconcile failed', {+ scope: 'GFS',+ gfs: gfsKey(gfs),+ err,+ })}}+ hccLogger.info('Resync pass completed', {+ scope: 'GFS',+ objects: totals.objects,+ writes: totals.writes,+ skips: totals.skips,+ passMs: Date.now() - started,+ })}async reconcileDelete(gfs: GlobalFileSystemCRD): Promise<void> {diff --git a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.tsindex 6dc85e953..7498b15db 100644--- a/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts+++ b/host-context-controller/src/k8s/gfsK8sApi.readFirst.test.ts@@ -1,5 +1,6 @@import { beforeEach, describe, expect, it, vi } from 'vitest'import type * as k8s from '@kubernetes/client-node'+import liveReaderFixture from '../__tests__/fixtures/629/gfsc-reader.deployment.json'import livePdbFixture from '../__tests__/fixtures/629/gfsc-writer-pdb.json'import { createsTotal, existenceReadsTotal, writeSkipsTotal } from '../metrics'import { podDisruptionBudgetMatchesDesired } from '../utils'@@ -34,7 +35,7 @@ const cases = [},{kind: 'Deployment',- updates: true,+ updates: false,body: {kind: 'Deployment',metadata,@@ -200,7 +201,7 @@ describe.each(cases)('K8sGfsApi $kind read-first', resource => {const h = harness(resource.kind)h.read.mockRejectedValueOnce({ code: 404 }).mockRejectedValueOnce({ code: 404 })h.create.mockRejectedValueOnce({ code: 409 })- await expect(apply(h.api)).resolves.toBeUndefined()+ await expect(apply(h.api)).resolves.toBe('missing')expect(h.read).toHaveBeenCalledTimes(2)expect(h.create).toHaveBeenCalledTimes(1)expect(h.replace).not.toHaveBeenCalled()@@ -345,3 +346,111 @@ describe('K8sGfsApi PodDisruptionBudget no-op gate (T4)', () => {expect(podDisruptionBudgetMatchesDesired(desired, live)).toBe(false)})})++const REVISION_ANNOTATION = 'deployment.kubernetes.io/revision'+const RESTARTED_AT = 'kubectl.kubernetes.io/restartedAt'++function desiredReaderFromFixture(): k8s.V1Deployment {+ const desired = structuredClone(liveReaderFixture) as k8s.V1Deployment+ if (desired.metadata?.annotations) {+ delete desired.metadata.annotations[REVISION_ANNOTATION]+ }+ return desired+}++function liveReader(overrides: Partial<k8s.V1Deployment['spec']> = {}): k8s.V1Deployment {+ const live = structuredClone(liveReaderFixture) as k8s.V1Deployment+ return {+ ...live,+ metadata: {+ ...live.metadata,+ resourceVersion: '1',+ uid: 'reader-uid',+ creationTimestamp: new Date('2026-01-01T00:00:00Z'),+ managedFields: [{ manager: 'kube-controller-manager', operation: 'Update' }],+ },+ spec: {+ ...live.spec,+ ...overrides,+ },+ status: { observedGeneration: 1, availableReplicas: live.spec?.replicas ?? 2 },+ }+}++describe('K8sGfsApi Deployment no-op gate (T3, T8)', () => {+ const namespace = 'gfs'++ async function deploymentSkipCount() {+ return (+ (await writeSkipsTotal.get()).values.find(row => row.labels.kind === 'Deployment')?.value ?? 0+ )+ }++ it('T3: skips replace when the live reader matches after revision merge', async () => {+ const desired = desiredReaderFromFixture()+ const equal = harness('Deployment')+ equal.read.mockImplementation(async () => {+ equal.events.push('GET')+ return liveReader()+ })+ await equal.api.applyDeployment(desired, namespace)+ expect(equal.read).toHaveBeenCalledTimes(1)+ expect(equal.replace).toHaveBeenCalledTimes(0)+ expect(await deploymentSkipCount()).toBe(1)+ })++ it('T3: writes when live replicas are 0 and restores the desired replica count', async () => {+ const desired = desiredReaderFromFixture()+ const scaled = harness('Deployment')+ scaled.read.mockImplementation(async () => {+ scaled.events.push('GET')+ return liveReader({ replicas: 0 })+ })+ await scaled.api.applyDeployment(desired, namespace)+ expect(scaled.read).toHaveBeenCalledTimes(1)+ expect(scaled.replace).toHaveBeenCalledTimes(1)+ const replaced = scaled.replace.mock.calls[0][0].body as k8s.V1Deployment+ expect(replaced.spec?.replicas).toBe(desired.spec?.replicas)+ expect(await deploymentSkipCount()).toBe(0)+ })++ it('T8: skips replace when live template has restartedAt and increments writeSkipsTotal', async () => {+ const desired = desiredReaderFromFixture()+ const restarted = harness('Deployment')+ const live = liveReader()+ const templateMeta = live.spec?.template?.metadata+ if (!templateMeta) throw new Error('expected pod template metadata')+ templateMeta.annotations = {+ ...templateMeta.annotations,+ [RESTARTED_AT]: '2026-09-16T20:33:45Z',+ }+ restarted.read.mockImplementation(async () => {+ restarted.events.push('GET')+ return live+ })+ await restarted.api.applyDeployment(desired, namespace)+ expect(restarted.read).toHaveBeenCalledTimes(1)+ expect(restarted.replace).toHaveBeenCalledTimes(0)+ expect(await deploymentSkipCount()).toBe(1)+ })++ it('T3: retries a drifted reader PUT after 409 using a fresh resourceVersion', async () => {+ const desired = desiredReaderFromFixture()+ const h = harness('Deployment')+ let version = 0+ h.read.mockImplementation(async () => {+ h.events.push('GET')+ const live = liveReader({ replicas: 0 })+ live.metadata = { ...live.metadata, resourceVersion: String(++version) }+ return live+ })+ h.replace.mockImplementationOnce(async ({ body }) => {+ h.events.push(`PUT:${body.metadata?.resourceVersion}`)+ throw { code: 409 }+ })+ await h.api.applyDeployment(desired, namespace)+ expect(h.events).toEqual(['GET', 'PUT:1', 'GET', 'PUT:2'])+ expect(h.replace).toHaveBeenCalledTimes(2)+ expect(h.create).not.toHaveBeenCalled()+ })+})diff --git a/host-context-controller/src/k8s/gfsK8sApi.ts b/host-context-controller/src/k8s/gfsK8sApi.tsindex 681ec5a1b..a82e093e6 100644--- a/host-context-controller/src/k8s/gfsK8sApi.ts+++ b/host-context-controller/src/k8s/gfsK8sApi.ts@@ -3,12 +3,15 @@ import type { GfsK8sApi } from '../gfsReconciler'import { createsTotal } from '../metrics'import type { GlobalFileSystemStatus } from '../types'import {+ type ResourceApplyResult,applyNetworkPolicy,+ deploymentMatchesDesired,ensureResource,getErrorCode,observeCreate,observeExistenceRead,podDisruptionBudgetMatchesDesired,+ preserveDeploymentAnnotations,preserveObjectAnnotations,replaceWithConflictRetry,} from '../utils'@@ -45,9 +48,12 @@ export class K8sGfsApi implements GfsK8sApi {}}- async applyPvc(pvc: k8s.V1PersistentVolumeClaim, namespace: string): Promise<void> {+ async applyPvc(+ pvc: k8s.V1PersistentVolumeClaim,+ namespace: string+ ): Promise<ResourceApplyResult> {const name = pvc.metadata?.name ?? ''- await ensureResource<k8s.V1PersistentVolumeClaim>({+ return ensureResource<k8s.V1PersistentVolumeClaim>({read: () =>observeExistenceRead('PersistentVolumeClaim', () =>this.coreApi.readNamespacedPersistentVolumeClaim({ name, namespace })@@ -78,17 +84,21 @@ export class K8sGfsApi implements GfsK8sApi {}}- async scaleDeployment(name: string, namespace: string, replicas: number): Promise<void> {+ async scaleDeployment(+ name: string,+ namespace: string,+ replicas: number+ ): Promise<ResourceApplyResult> {let existing: k8s.V1Deploymenttry {existing = await this.appsApi.readNamespacedDeployment({ name, namespace })} catch (err) {- if (getErrorCode(err) === 404) return+ if (getErrorCode(err) === 404) return 'missing'throw err}- if ((existing.spec?.replicas ?? 0) === replicas) return+ if ((existing.spec?.replicas ?? 0) === replicas) return 'up_to_date'- await replaceWithConflictRetry<k8s.V1Deployment>({+ return replaceWithConflictRetry<k8s.V1Deployment>({description: `deployment "${name}" scale in ${namespace}`,logPrefix: LOG,body: existing,@@ -106,9 +116,9 @@ export class K8sGfsApi implements GfsK8sApi {})}- async applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<void> {+ async applyDeployment(dep: k8s.V1Deployment, namespace: string): Promise<ResourceApplyResult> {const name = dep.metadata?.name ?? ''- await ensureResource<k8s.V1Deployment>({+ return ensureResource<k8s.V1Deployment>({read: () =>observeExistenceRead('Deployment', () =>this.appsApi.readNamespacedDeployment({ name, namespace })@@ -125,13 +135,18 @@ export class K8sGfsApi implements GfsK8sApi {body: dep,read,replace: body => this.appsApi.replaceNamespacedDeployment({ name, namespace, body }),+ mergeExisting: preserveDeploymentAnnotations,+ isUpToDate: deploymentMatchesDesired,}),})}- async applyPodDisruptionBudget(pdb: k8s.V1PodDisruptionBudget, namespace: string): Promise<void> {+ async applyPodDisruptionBudget(+ pdb: k8s.V1PodDisruptionBudget,+ namespace: string+ ): Promise<ResourceApplyResult> {const name = pdb.metadata?.name ?? ''- await ensureResource<k8s.V1PodDisruptionBudget>({+ return ensureResource<k8s.V1PodDisruptionBudget>({read: () =>observeExistenceRead('PodDisruptionBudget', () =>this.policyApi.readNamespacedPodDisruptionBudget({ name, namespace })@@ -155,9 +170,9 @@ export class K8sGfsApi implements GfsK8sApi {})}- async applyService(svc: k8s.V1Service, namespace: string): Promise<void> {+ async applyService(svc: k8s.V1Service, namespace: string): Promise<ResourceApplyResult> {const name = svc.metadata?.name ?? ''- await ensureResource<k8s.V1Service>({+ return ensureResource<k8s.V1Service>({read: () =>observeExistenceRead('Service', () =>this.coreApi.readNamespacedService({ name, namespace })@@ -173,8 +188,11 @@ export class K8sGfsApi implements GfsK8sApi {})}- async applyNetworkPolicy(np: k8s.V1NetworkPolicy, namespace: string): Promise<void> {- await applyNetworkPolicy(this.networkingApi, np.metadata?.name ?? '', namespace, np, LOG)+ async applyNetworkPolicy(+ np: k8s.V1NetworkPolicy,+ namespace: string+ ): Promise<ResourceApplyResult> {+ return applyNetworkPolicy(this.networkingApi, np.metadata?.name ?? '', namespace, np, LOG)}async isDeploymentAvailable(name: string, namespace: string): Promise<boolean> {diff --git a/host-context-controller/src/llmHookReconciler.test.ts b/host-context-controller/src/llmHookReconciler.test.tsindex 9a55f1462..ac83bd2e9 100644--- a/host-context-controller/src/llmHookReconciler.test.ts+++ b/host-context-controller/src/llmHookReconciler.test.ts@@ -21,6 +21,7 @@ import {podKeyResourceName,referencedHookIds,} from './llmHookReconciler'+import { hccLogger } from './logger'import { HostCRD, LlmHookCRD } from './types'vi.mock('./config', () => ({@@ -1032,4 +1033,134 @@ describe('LlmHookReconciler', () => {expect(new Set(referencedHookIds(host))).toEqual(new Set(['x', 'y', 'x']))expect(referencedHookIds(undefined)).toEqual([])})++ describe('fullReconcile resync pass log (D3)', () => {+ function lastBody<T>(fn: { mock: { calls: unknown[][] } }): T | undefined {+ const call = fn.mock.calls.at(-1)?.[0] as { body?: T } | undefined+ return call?.body+ }++ function resyncCalls() {+ return vi+ .mocked(hccLogger.info)+ .mock.calls.filter(([message]) => message === 'Resync pass completed')+ }++ async function seedMatchingHook() {+ const hook = makeHook({+ name: 'd3',+ spec: { target: { image: { ref: IMG, port: 8080 } }, lifecyclePoints: ['preCall'] },+ })+ hooks.set('d3', hook)+ await reconciler.fullReconcile([hook])+ const dep =+ lastBody<k8s.V1Deployment>(appsApi.replaceNamespacedDeployment) ??+ lastBody<k8s.V1Deployment>(appsApi.createNamespacedDeployment)+ const svc =+ lastBody<k8s.V1Service>(coreApi.replaceNamespacedService) ??+ lastBody<k8s.V1Service>(coreApi.createNamespacedService)+ const np =+ lastBody<k8s.V1NetworkPolicy>(networkingApi.replaceNamespacedNetworkPolicy) ??+ lastBody<k8s.V1NetworkPolicy>(networkingApi.createNamespacedNetworkPolicy)+ if (!dep || !svc || !np) throw new Error('expected LlmHook resources to be written')+ appsApi.readNamespacedDeployment.mockResolvedValue({+ ...dep,+ metadata: { ...dep.metadata, resourceVersion: '1' },+ status: { readyReplicas: 1 },+ })+ coreApi.readNamespacedService.mockResolvedValue({+ ...svc,+ metadata: { ...svc.metadata, resourceVersion: '1' },+ spec: { ...svc.spec, clusterIP: '10.0.0.1' },+ })+ networkingApi.readNamespacedNetworkPolicy.mockResolvedValue({+ ...np,+ metadata: { ...np.metadata, resourceVersion: '1' },+ })+ appsApi.replaceNamespacedDeployment.mockClear()+ coreApi.replaceNamespacedService.mockClear()+ networkingApi.replaceNamespacedNetworkPolicy.mockClear()+ return hook+ }++ it('D3: an already-synced LlmHook pass logs writes 0 and skips equal to objects', async () => {+ const hook = await seedMatchingHook()+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})+ await reconciler.fullReconcile([hook])+ const line = resyncCalls().at(-1)+ expect(line?.[0]).toBe('Resync pass completed')+ expect(line?.[1]).toEqual(+ expect.objectContaining({+ scope: 'LlmHook',+ writes: 0,+ objects: expect.any(Number),+ skips: expect.any(Number),+ passMs: expect.any(Number),+ })+ )+ const fields = line?.[1] as { objects: number; skips: number }+ expect(fields.skips).toBe(fields.objects)+ expect(fields.objects).toBeGreaterThan(0)+ })++ it('D3: a drifted LlmHook Deployment logs writes 1', async () => {+ const hook = await seedMatchingHook()+ const live = await appsApi.readNamespacedDeployment()+ appsApi.readNamespacedDeployment.mockResolvedValue({+ ...live,+ spec: {+ ...live.spec,+ template: {+ ...live.spec?.template,+ spec: {+ ...live.spec?.template?.spec,+ containers: [{ name: 'hook', image: 'registry.example.com/other:drift' }],+ },+ },+ },+ })+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})+ await reconciler.fullReconcile([hook])+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ const line = resyncCalls().at(-1)+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'LlmHook', writes: 1 }))+ })++ it('D3: a concurrent watch reconcile does not change the resync pass counters', async () => {+ const hook = await seedMatchingHook()+ let releaseSweep: (() => void) | undefined+ const sweepHold = new Promise<void>(resolve => {+ releaseSweep = resolve+ })+ let sweepStarted = false+ appsApi.listNamespacedDeployment.mockImplementation(async () => {+ sweepStarted = true+ await sweepHold+ return { items: [] }+ })+ const info = vi.spyOn(hccLogger, 'info').mockImplementation(() => {})+ const pass = reconciler.fullReconcile([hook])+ await vi.waitFor(() => expect(sweepStarted).toBe(true))+ const live = await appsApi.readNamespacedDeployment()+ appsApi.readNamespacedDeployment.mockResolvedValue({+ ...live,+ spec: {+ ...live.spec,+ template: {+ ...live.spec?.template,+ spec: {+ ...live.spec?.template?.spec,+ containers: [{ name: 'hook', image: 'registry.example.com/watch:drift' }],+ },+ },+ },+ })+ await reconciler.reconcile(hook)+ expect(appsApi.replaceNamespacedDeployment).toHaveBeenCalledTimes(1)+ releaseSweep?.()+ await pass+ const line = resyncCalls().at(-1)+ expect(line?.[1]).toEqual(expect.objectContaining({ scope: 'LlmHook', writes: 0 }))+ })+ })})diff --git a/host-context-controller/src/llmHookReconciler.ts b/host-context-controller/src/llmHookReconciler.tsindex ecaa605de..8540cde7e 100644--- a/host-context-controller/src/llmHookReconciler.ts+++ b/host-context-controller/src/llmHookReconciler.ts@@ -23,6 +23,7 @@import * as k8s from '@kubernetes/client-node'import { IntOrString } from '@kubernetes/client-node/dist/types.js'import { createHash } from 'crypto'+import { AsyncLocalStorage } from 'node:async_hooks'import * as dns from 'node:dns/promises'import { isIP } from 'node:net'import { isWorkflowRecipeDefaultAllowedCapability } from '@clerum/workflow-recipe-capability-policy'@@ -40,6 +41,7 @@ import { createsTotal } from './metrics'import { isAllowedExternalEgressCidr, isPublicDnsHostname } from './networkPolicyReconciler'import { HostCRD, LlmHookCRD, LlmHookCondition, LlmHookImageTarget, LlmHookStatus } from './types'import {+ type ResourceApplyResult,deploymentMatchesDesired,ensureResource,getErrorCode,@@ -53,6 +55,18 @@ import {serviceMatchesDesired,} from './utils'+type LlmHookResyncCounters = { objects: number; writes: number; skips: number }++const llmHookResyncPass = new AsyncLocalStorage<LlmHookResyncCounters>()++function recordLlmHookApply(result: ResourceApplyResult): void {+ const acc = llmHookResyncPass.getStore()+ if (!acc) return+ acc.objects += 1+ if (result === 'created' || result === 'replaced') acc.writes += 1+ else if (result === 'up_to_date') acc.skips += 1+}+const GROUP = 'clerum.io'const VERSION = 'v1alpha1'const PLURAL_LLMHOOKS = 'llmhooks'@@ -299,27 +313,37 @@ export class LlmHookReconciler {* `llmhook-*` workloads whose pod key has zero live members (§3).*/async fullReconcile(hooks: LlmHookCRD[]): Promise<void> {- console.log(`${LOG} Running full reconciliation for ${hooks.length} LlmHook(s)`)- const imagePodKeys = new Set<string>()- for (const hook of hooks) {- const kind = classifyTarget(hook)- if (kind === 'image') {- const key = computePodKey(hook)- if (key) imagePodKeys.add(key)- } else {- await this.reconcile(hook)+ const started = Date.now()+ const acc: LlmHookResyncCounters = { objects: 0, writes: 0, skips: 0 }+ hccLogger.info('Running full reconciliation', { scope: 'LlmHook', hooks: hooks.length })+ await llmHookResyncPass.run(acc, async () => {+ const imagePodKeys = new Set<string>()+ for (const hook of hooks) {+ const kind = classifyTarget(hook)+ if (kind === 'image') {+ const key = computePodKey(hook)+ if (key) imagePodKeys.add(key)+ } else {+ await this.reconcile(hook)+ }}- }- for (const podKey of imagePodKeys) {- await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))- }- // Converge every Host's scoped egress-to-hooks policy (N1/N7) — the periodic- // backstop for a missed watch event or an image bump that moved a pod key.- for (const host of this.hosts.values()) {- await this.reconcileHostEgress(host)- }- await this.sweepOrphanedWorkloads()- console.log(`${LOG} Full reconciliation complete`)+ for (const podKey of imagePodKeys) {+ await this.runSerialized(podKey, () => this.reconcilePodKey(podKey))+ }+ // Converge every Host's scoped egress-to-hooks policy (N1/N7) — the periodic+ // backstop for a missed watch event or an image bump that moved a pod key.+ for (const host of this.hosts.values()) {+ await this.reconcileHostEgress(host)+ }+ await this.sweepOrphanedWorkloads()+ })+ hccLogger.info('Resync pass completed', {+ scope: 'LlmHook',+ objects: acc.objects,+ writes: acc.writes,+ skips: acc.skips,+ passMs: Date.now() - started,+ })}// ─── Members / reverse index ────────────────────────────────────────@@ -603,7 +627,7 @@ export class LlmHookReconciler {// workflow-recipe allowlist, but strip anything outside it at reconcile too.const addCapabilities = (img.security?.addCapabilities ?? []).filter(cap => {if (isWorkflowRecipeDefaultAllowedCapability(cap)) return true- console.warn(`${LOG} pod key ${podKey}: capability "${cap}" stripped (forbidden)`)+ hccLogger.warn('Forbidden capability stripped', { scope: LOG, podKey, capability: cap })return false})@@ -759,87 +783,91 @@ export class LlmHookReconciler {): Promise<void> {const deployment = this.buildDeployment(podKey, members, credentialsRevision)const name = deployment.metadata!.name!- await ensureResource({- read: () =>- observeExistenceRead('Deployment', () =>- this.appsApi.readNamespacedDeployment({- name,+ recordLlmHookApply(+ await ensureResource({+ read: () =>+ observeExistenceRead('Deployment', () =>+ this.appsApi.readNamespacedDeployment({+ name,+ namespace: config.llmHooksNamespace,+ })+ ),+ create: async () => {+ await observeCreate('Deployment', () =>+ this.appsApi.createNamespacedDeployment({+ namespace: config.llmHooksNamespace,+ body: deployment,+ })+ )+ hccLogger.info('Deployment created', {+ scope: LOG,+ deployment: name,namespace: config.llmHooksNamespace,})- ),- create: async () => {- await observeCreate('Deployment', () =>- this.appsApi.createNamespacedDeployment({- namespace: config.llmHooksNamespace,+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `Deployment "${name}"`,+ logPrefix: LOG,body: deployment,- })- )- hccLogger.info('Deployment created', {- scope: LOG,- deployment: name,- namespace: config.llmHooksNamespace,- })- },- converge: read =>- replaceWithConflictRetry({- description: `Deployment "${name}"`,- logPrefix: LOG,- body: deployment,- mergeExisting: preserveDeploymentAnnotations,- isUpToDate: deploymentMatchesDesired,- read,- replace: body =>- this.appsApi.replaceNamespacedDeployment({- name,- namespace: config.llmHooksNamespace,- body,- }),- }),- onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),- })+ mergeExisting: preserveDeploymentAnnotations,+ isUpToDate: deploymentMatchesDesired,+ read,+ replace: body =>+ this.appsApi.replaceNamespacedDeployment({+ name,+ namespace: config.llmHooksNamespace,+ body,+ }),+ }),+ onSkipped: () => createsTotal.inc({ kind: 'Deployment', outcome: 'skipped' }),+ })+ )}private async ensureService(podKey: string, port: number): Promise<void> {const service = this.buildService(podKey, port)const name = service.metadata!.name!- await ensureResource({- read: () =>- observeExistenceRead('Service', () =>- this.coreApi.readNamespacedService({- name,+ recordLlmHookApply(+ await ensureResource({+ read: () =>+ observeExistenceRead('Service', () =>+ this.coreApi.readNamespacedService({+ name,+ namespace: config.llmHooksNamespace,+ })+ ),+ create: async () => {+ await observeCreate('Service', () =>+ this.coreApi.createNamespacedService({+ namespace: config.llmHooksNamespace,+ body: service,+ })+ )+ hccLogger.info('Service created', {+ scope: LOG,+ service: name,namespace: config.llmHooksNamespace,})- ),- create: async () => {- await observeCreate('Service', () =>- this.coreApi.createNamespacedService({- namespace: config.llmHooksNamespace,+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `Service "${name}"`,+ logPrefix: LOG,body: service,- })- )- hccLogger.info('Service created', {- scope: LOG,- service: name,- namespace: config.llmHooksNamespace,- })- },- converge: read =>- replaceWithConflictRetry({- description: `Service "${name}"`,- logPrefix: LOG,- body: service,- mergeExisting: preserveServiceAssignedFields,- isUpToDate: serviceMatchesDesired,- read,- replace: body =>- this.coreApi.replaceNamespacedService({- name,- namespace: config.llmHooksNamespace,- body,- }),- }),- onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),- })+ mergeExisting: preserveServiceAssignedFields,+ isUpToDate: serviceMatchesDesired,+ read,+ replace: body =>+ this.coreApi.replaceNamespacedService({+ name,+ namespace: config.llmHooksNamespace,+ body,+ }),+ }),+ onSkipped: () => createsTotal.inc({ kind: 'Service', outcome: 'skipped' }),+ })+ )}private async ensureNetworkPolicy(@@ -854,30 +882,32 @@ export class LlmHookReconciler {private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy): Promise<void> {const name = policy.metadata!.name!const namespace = policy.metadata!.namespace ?? config.llmHooksNamespace- await ensureResource({- read: () =>- observeExistenceRead('NetworkPolicy', () =>- this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })- ),- create: async () => {- await observeCreate('NetworkPolicy', () =>- this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })- )- hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })- },- converge: read =>- replaceWithConflictRetry({- description: `NetworkPolicy "${name}"`,- logPrefix: LOG,- body: policy,- mergeExisting: preserveObjectAnnotations,- isUpToDate: networkPolicyMatchesDesired,- read,- replace: body =>- this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),- }),- onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),- })+ recordLlmHookApply(+ await ensureResource({+ read: () =>+ observeExistenceRead('NetworkPolicy', () =>+ this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })+ ),+ create: async () => {+ await observeCreate('NetworkPolicy', () =>+ this.networkingApi.createNamespacedNetworkPolicy({ namespace, body: policy })+ )+ hccLogger.info('NetworkPolicy created', { scope: LOG, policy: name, namespace })+ },+ converge: read =>+ replaceWithConflictRetry({+ description: `NetworkPolicy "${name}"`,+ logPrefix: LOG,+ body: policy,+ mergeExisting: preserveObjectAnnotations,+ isUpToDate: networkPolicyMatchesDesired,+ read,+ replace: body =>+ this.networkingApi.replaceNamespacedNetworkPolicy({ name, namespace, body }),+ }),+ onSkipped: () => createsTotal.inc({ kind: 'NetworkPolicy', outcome: 'skipped' }),+ })+ )}/**@@ -894,9 +924,12 @@ export class LlmHookReconciler {const svc = hook.spec.target?.serviceif (!svc?.name || !svc.namespace || !svc.port) returnif (svc.namespace !== config.llmHooksNamespace) {- console.warn(- `${LOG} service-target hook "${hook.name}" → Service ${svc.namespace}/${svc.name} outside ${config.llmHooksNamespace}; ingress not enforced by HCC`- )+ hccLogger.warn('Service-target hook points outside llm-hooks; ingress not enforced', {+ scope: LOG,+ hook: hook.name,+ serviceNamespace: svc.namespace,+ service: svc.name,+ })return}@@ -905,9 +938,12 @@ export class LlmHookReconciler {// fail-closed under the namespace default-deny instead.const selector = await this.readServiceSelector(svc.name, svc.namespace)if (!selector) {- console.warn(- `${LOG} service-target hook "${hook.name}": Service ${svc.namespace}/${svc.name} missing or selector-less; leaving fail-closed under default-deny`- )+ hccLogger.warn('Service-target hook missing selector; leaving fail-closed', {+ scope: LOG,+ hook: hook.name,+ serviceNamespace: svc.namespace,+ service: svc.name,+ })return}@@ -958,9 +994,12 @@ export class LlmHookReconciler {const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })- console.log(`${LOG} Deleted NetworkPolicy "${name}"`)+ hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name })} else {- console.warn(`${LOG} Skipping NetworkPolicy "${name}" delete — not HCC-owned`)+ hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {+ scope: LOG,+ policy: name,+ })}} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -1085,7 +1124,7 @@ export class LlmHookReconciler {const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })if (np.metadata?.labels?.[MANAGED_BY_LABEL] === MANAGED_BY_VALUE) {await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })- console.log(`${LOG} Deleted NetworkPolicy "${name}" (${ns})`)+ hccLogger.info('NetworkPolicy deleted', { scope: LOG, policy: name, namespace: ns })}} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -1110,9 +1149,12 @@ export class LlmHookReconciler {const dep = await this.appsApi.readNamespacedDeployment({ name, namespace: ns })if (this.isHccOwned(dep, podKey)) {await this.appsApi.deleteNamespacedDeployment({ name, namespace: ns })- console.log(`${LOG} Deleted Deployment "${name}" (0 members)`)+ hccLogger.info('Deployment deleted (0 members)', { scope: LOG, deployment: name })} else {- console.warn(`${LOG} Skipping Deployment "${name}" delete — not HCC-owned`)+ hccLogger.warn('Skipping Deployment delete — not HCC-owned', {+ scope: LOG,+ deployment: name,+ })}} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -1122,9 +1164,9 @@ export class LlmHookReconciler {const svc = await this.coreApi.readNamespacedService({ name, namespace: ns })if (this.isHccOwned(svc, podKey)) {await this.coreApi.deleteNamespacedService({ name, namespace: ns })- console.log(`${LOG} Deleted Service "${name}" (0 members)`)+ hccLogger.info('Service deleted (0 members)', { scope: LOG, service: name })} else {- console.warn(`${LOG} Skipping Service "${name}" delete — not HCC-owned`)+ hccLogger.warn('Skipping Service delete — not HCC-owned', { scope: LOG, service: name })}} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -1134,9 +1176,12 @@ export class LlmHookReconciler {const np = await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace: ns })if (this.isHccOwned(np, podKey)) {await this.networkingApi.deleteNamespacedNetworkPolicy({ name, namespace: ns })- console.log(`${LOG} Deleted NetworkPolicy "${name}" (0 members)`)+ hccLogger.info('NetworkPolicy deleted (0 members)', { scope: LOG, policy: name })} else {- console.warn(`${LOG} Skipping NetworkPolicy "${name}" delete — not HCC-owned`)+ hccLogger.warn('Skipping NetworkPolicy delete — not HCC-owned', {+ scope: LOG,+ policy: name,+ })}} catch (error) {if (getErrorCode(error) !== 404) throw error@@ -1158,14 +1203,14 @@ export class LlmHookReconciler {})deployments = resp.items ?? []} catch (error) {- console.error(`${LOG} Orphan sweep: failed to list Deployments:`, error)+ hccLogger.error('Orphan sweep failed to list Deployments', { scope: LOG, err: error })return}for (const dep of deployments) {const podKey = dep.metadata?.labels?.[HOOK_PODKEY_LABEL]if (!podKey) continueif (this.membersForPodKey(podKey).length > 0) continue- console.log(`${LOG} Orphan sweep: pod key ${podKey} has 0 members — deleting workload`)+ hccLogger.info('Orphan sweep deleting workload with 0 members', { scope: LOG, podKey })await this.runSerialized(podKey, () => this.gcPodKey(podKey))}}@@ -1211,7 +1256,7 @@ export class LlmHookReconciler {}}} catch (error) {- console.warn(`${LOG} Failed to read observedDigest for pod key ${podKey}:`, error)+ hccLogger.warn('Failed to read observedDigest', { scope: LOG, podKey, err: error })}return ''}@@ -1243,7 +1288,7 @@ export class LlmHookReconciler {})) as { metadata?: { resourceVersion?: string }; status?: LlmHookStatus }} catch (error) {if (getErrorCode(error) === 404) return- console.warn(`${LOG} Failed to read status for "${hook.name}":`, error)+ hccLogger.warn('Failed to read LlmHook status', { scope: LOG, hook: hook.name, err: error })return}@@ -1318,7 +1363,11 @@ export class LlmHookReconciler {if (getErrorCode(error) === 404) returnconst code = getErrorCode(error)if ((code === 409 || code === 422) && attempt < MAX_ATTEMPTS) continue- console.warn(`${LOG} Failed to write status on "${hook.name}":`, error)+ hccLogger.warn('Failed to write LlmHook status', {+ scope: LOG,+ hook: hook.name,+ err: error,+ })return}}diff --git a/host-context-controller/src/utils.ts b/host-context-controller/src/utils.tsindex 2b1989900..7fcb2aacb 100644--- a/host-context-controller/src/utils.ts+++ b/host-context-controller/src/utils.ts@@ -27,6 +27,9 @@ export async function observeCreate<T>(kind: CreateKind, create: () => Promise<T}}+/** Outcome of a read-first create-or-replace. Callers may ignore this value. */+export type ResourceApplyResult = 'created' | 'replaced' | 'up_to_date' | 'missing' | 'not_allowed'+/** Observe one existence GET without changing its value or error handling. */export async function observeExistenceRead<T>(kind: CreateKind,@@ -53,12 +56,12 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>create: () => Promise<unknown>existing?: T | null/** False preserves a caller's non-throwing failure without counting a skip. */- converge: (read: () => Promise<T>, observed?: T) => Promise<void | false>+ converge: (read: () => Promise<T>, observed?: T) => Promise<void | false | ResourceApplyResult>onSkipped: () => voidmutationAllowed?: () => boolean-}): Promise<void> {+}): Promise<ResourceApplyResult> {const { read, create, converge, onSkipped, mutationAllowed } = opts- if (mutationAllowed && !mutationAllowed()) return+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'const validate = (value: T): T => {if (!value || typeof value.metadata?.name !== 'string' || !value.metadata.name.trim()) {@@ -92,7 +95,7 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>} else if (existing !== null) {validate(existing)}- if (mutationAllowed && !mutationAllowed()) return+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'if (existing !== null) {let snapshot: T | undefined = existing@@ -108,16 +111,22 @@ export async function ensureResource<T extends { metadata?: { name?: string } }>if (result !== false && latestReadSucceeded && (!mutationAllowed || mutationAllowed())) {onSkipped()}- return+ if (result === false) return 'not_allowed'+ if (typeof result === 'string') return result+ return latestReadSucceeded ? 'up_to_date' : 'missing'}try {await create()+ return 'created'} catch (error) {if (error == null || getErrorCode(error) !== 409) throw error- if (mutationAllowed && !mutationAllowed()) return+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'// Absence observed before the POST cannot survive a create conflict.- await converge(readFresh)+ const result = await converge(readFresh)+ if (result === false) return 'not_allowed'+ if (typeof result === 'string') return result+ return latestReadSucceeded ? 'up_to_date' : 'missing'}}@@ -161,7 +170,7 @@ export async function replaceWithConflictRetry</** Rebuild desired state before each attempt without an unused eager body. */| { body?: never; resolveBody: () => T | Promise<T> })-): Promise<void> {+): Promise<ResourceApplyResult> {const {description,logPrefix,@@ -185,7 +194,7 @@ export async function replaceWithConflictRetry<scope: logPrefix,description,})- return+ return 'missing'}throw err}@@ -198,16 +207,16 @@ export async function replaceWithConflictRetry<}const next = mergeExisting ? mergeExisting(base, existing) : baseif (isUpToDate?.(next, existing)) {- if (mutationAllowed && !mutationAllowed()) return+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'writeSkipsTotal.inc({ kind: next.kind ?? 'unknown' })- return+ return 'up_to_date'}- if (mutationAllowed && !mutationAllowed()) return+ if (mutationAllowed && !mutationAllowed()) return 'not_allowed'try {await replace(next)writesTotal.inc({ kind: next.kind ?? 'unknown' })hccLogger.info('Kubernetes resource updated', { scope: logPrefix, description, attempt })- return+ return 'replaced'} catch (err) {if (getErrorCode(err) === 409 && attempt < maxAttempts) {// Jittered backoff: 25–125ms on attempt 1, 50–250ms on attempt 2.@@ -219,6 +228,7 @@ export async function replaceWithConflictRetry<throw err}}+ throw new Error(`replaceWithConflictRetry exhausted attempts for ${description}`)}function mergeAnnotations(@@ -605,8 +615,8 @@ export async function applyNetworkPolicy(validateExisting?: (existing: k8s.V1NetworkPolicy) => void,missingIsError = false,existing?: k8s.V1NetworkPolicy | null-): Promise<void> {- await ensureResource<k8s.V1NetworkPolicy>({+): Promise<ResourceApplyResult> {+ return ensureResource<k8s.V1NetworkPolicy>({existing,mutationAllowed,read: () =>