Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(hcc): roll mcp-host when the guardrails block is removed

Andy Paviad612a5674e58
diff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.json
index 25be4c4da..0445983f3 100644
--- a/host-context-controller/package-lock.json
+++ b/host-context-controller/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.175",
+ "version": "0.1.176",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-host-context-controller",
- "version": "0.1.175",
+ "version": "0.1.176",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/host-context-controller/package.json b/host-context-controller/package.json
index ae68fe707..50c3259a2 100644
--- a/host-context-controller/package.json
+++ b/host-context-controller/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-host-context-controller",
- "version": "0.1.175",
+ "version": "0.1.176",
"description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API",
"main": "dist/main.js",
"engines": {
diff --git a/host-context-controller/src/__tests__/hostDeploymentAnnotationMerge.test.ts b/host-context-controller/src/__tests__/hostDeploymentAnnotationMerge.test.ts
new file mode 100644
index 000000000..10a2692e8
--- /dev/null
+++ b/host-context-controller/src/__tests__/hostDeploymentAnnotationMerge.test.ts
@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
+import { preserveDeploymentAnnotationsExcept } from '../hostReconciler'
+
+const GUARDRAILS = 'clerum.io/guardrails-revision'
+const RESTARTED_AT = 'kubectl.kubernetes.io/restartedAt'
+
+function deployment(templateAnnotations?: Record<string, string>): k8s.V1Deployment {
+ return {
+ apiVersion: 'apps/v1',
+ kind: 'Deployment',
+ metadata: { name: 'chatllm', namespace: 'mcp-host' },
+ spec: {
+ selector: { matchLabels: { app: 'chatllm' } },
+ template: {
+ metadata: { ...(templateAnnotations ? { annotations: templateAnnotations } : {}) },
+ spec: { containers: [] },
+ },
+ },
+ }
+}
+
+const merge = (desired: k8s.V1Deployment, existing: k8s.V1Deployment) =>
+ preserveDeploymentAnnotationsExcept(desired, existing, [GUARDRAILS])?.spec?.template?.metadata
+ ?.annotations
+
+describe('preserveDeploymentAnnotationsExcept — pod-template merge', () => {
+ // T5 regression. `mergeAnnotations` is {...existing, ...desired}, so before this
+ // an omitted controller-owned key kept its live value: dropping spec.guardrails
+ // produced a pod template byte-identical to the running one, deploymentMatches-
+ // Desired returned true, and replaceWithConflictRetry skipped the write. The
+ // agent kept enforcing guardrails the operator had uninstalled.
+ it('drops a controller-owned annotation the desired template omits', () => {
+ const merged = merge(deployment(), deployment({ [GUARDRAILS]: 'abc123' }))
+
+ expect(merged?.[GUARDRAILS]).toBeUndefined()
+ })
+
+ it('produces a pod-template diff on removal, so the pod rolls', () => {
+ const existing = deployment({ [GUARDRAILS]: 'abc123' })
+ const merged = merge(deployment(), existing)
+
+ expect(merged).not.toEqual(existing.spec?.template?.metadata?.annotations)
+ })
+
+ it('keeps the desired value when guardrails are present', () => {
+ const merged = merge(deployment({ [GUARDRAILS]: 'new' }), deployment({ [GUARDRAILS]: 'old' }))
+
+ expect(merged?.[GUARDRAILS]).toBe('new')
+ })
+
+ it('preserves operator/runtime annotations HCC does not author', () => {
+ const merged = merge(
+ deployment(),
+ deployment({ [GUARDRAILS]: 'abc123', [RESTARTED_AT]: '2026-08-18T10:00:00Z' })
+ )
+
+ expect(merged?.[RESTARTED_AT]).toBe('2026-08-18T10:00:00Z')
+ expect(merged?.[GUARDRAILS]).toBeUndefined()
+ })
+
+ it('leaves annotations undefined when the drop empties the map', () => {
+ const merged = merge(deployment(), deployment({ [GUARDRAILS]: 'abc123' }))
+
+ expect(merged).toBeUndefined()
+ })
+
+ it('is a no-op when neither side carries the controller-owned key', () => {
+ const merged = merge(deployment({ [RESTARTED_AT]: 'x' }), deployment())
+
+ expect(merged).toEqual({ [RESTARTED_AT]: 'x' })
+ })
+})
diff --git a/host-context-controller/src/hostReconciler.ts b/host-context-controller/src/hostReconciler.ts
index 3ed4e9dcc..94e338245 100644
--- a/host-context-controller/src/hostReconciler.ts
+++ b/host-context-controller/src/hostReconciler.ts
@@ -194,12 +194,14 @@ export type ResolveContextMountsFn = (host: HostCRD) => Promise<ResolvedSfsMount
const CONTEXT_LABEL = 'clerum.io/context'
const CONTEXT_MOUNT_PATH_PATTERN = /^\/[a-zA-Z0-9_.][a-zA-Z0-9_.\/-]*$/
const RUNTIME_TOKEN_REVISION_ANNOTATION = 'clerum.io/runtime-token-revision'
-// Rolls the mcp-host pod when Host.spec.guardrails changes. mcp-host reads the
-// guardrails block (installed-hook refs, built-ins, limits) ONLY at boot, so an
-// install/uninstall/upgrade of a hook — or any edit to the block — would
-// otherwise not reach the running agent until the pod happened to restart.
-// Stamping a hash of the guardrails spec onto the pod template makes any change
-// flip the template → rolling restart → mcp-host re-reads guardrails at boot.
+// Rolls the mcp-host pod when Host.spec.guardrails changes. mcp-host re-resolves
+// the guardrails block (installed-hook refs, built-ins, limits) live off its Host
+// watch, but a watch that has lapsed leaves the running agent on the block it last
+// saw; the roll is the delivery path that does not depend on that watch. Stamping
+// a hash of the guardrails spec onto the pod template makes any change flip the
+// template → rolling restart → mcp-host re-reads guardrails at boot. Removal is
+// handled by preserveHostDeploymentAnnotations, since an omitted key would
+// otherwise survive the annotation merge and produce no diff.
// Mirrors the runtime-token / credentials revision annotations.
const GUARDRAILS_REVISION_ANNOTATION = 'clerum.io/guardrails-revision'
@@ -2386,11 +2388,13 @@ export class HostReconciler {
if (runtimeTokenRevision) {
podAnnotations[RUNTIME_TOKEN_REVISION_ANNOTATION] = runtimeTokenRevision
}
- // Roll the mcp-host pod whenever the guardrails block changes: mcp-host reads
- // Host.spec.guardrails only at boot, so without a pod-template diff an
- // installed-hook change never reaches the running agent. Stamp a hash of the
- // block — present→changed→removed all flip the template and trigger a rollout
- // (when guardrails is absent the annotation is simply omitted).
+ // Roll the mcp-host pod whenever the guardrails block changes. mcp-host does
+ // re-resolve guardrails live, but only while its Host watch is connected, so
+ // the roll is what delivers the change to an agent whose watch has lapsed.
+ // Stamp a hash of the block: present→changed flips the value, and removal
+ // omits the key — which only reaches the pod because
+ // preserveHostDeploymentAnnotations strips it from the merge instead of
+ // letting the live value survive (an omitted key alone produces no diff).
if (host.spec.guardrails) {
podAnnotations[GUARDRAILS_REVISION_ANNOTATION] = createHash('sha256')
.update(deepStableStringify(host.spec.guardrails))
@@ -2879,7 +2883,7 @@ export class HostReconciler {
logPrefix: '[HostReconciler]',
body: deployment,
resolveBody: buildDesiredDeployment,
- mergeExisting: preserveDeploymentAnnotations,
+ mergeExisting: preserveHostDeploymentAnnotations,
isUpToDate: deploymentMatchesDesired,
read: () =>
this.appsApi.readNamespacedDeployment({
@@ -4336,25 +4340,32 @@ function deploymentMatchesDesired(desired: k8s.V1Deployment, existing: k8s.V1Dep
}
/**
- * Preserve operational annotations without retaining the channel-reader
- * revision when HCC intentionally omits it from the desired pod template.
- * That annotation is controller-owned and must be cleared when no backing
- * CommunicationChannel Secret is resolvable.
+ * Preserve operational pod-template annotations (kubectl restart markers,
+ * operator edits) while keeping CONTROLLER-OWNED ones authoritative: a key in
+ * `controllerOwned` that the desired template omits is REMOVED from the merge
+ * rather than surviving from the live object.
+ *
+ * Without this, "the field went away" cannot be expressed at all. The merge is
+ * `{...existing, ...desired}`, so an omitted key keeps its old value, the merged
+ * object then compares equal to what is live, and `replaceWithConflictRetry`
+ * skips the write entirely — no template diff, no rolling restart, no write.
*/
-function preserveChannelReaderDeploymentAnnotations(
+export function preserveDeploymentAnnotationsExcept(
desired: k8s.V1Deployment,
- existing: k8s.V1Deployment
+ existing: k8s.V1Deployment,
+ controllerOwned: readonly string[]
): k8s.V1Deployment {
const preserved = preserveDeploymentAnnotations(desired, existing)
const desiredAnnotations = desired.spec?.template?.metadata?.annotations
- if (desiredAnnotations?.['clerum.io/credentials-revision'] !== undefined) return preserved
+ const dropped = controllerOwned.filter(key => desiredAnnotations?.[key] === undefined)
+ if (dropped.length === 0) return preserved
const spec = preserved.spec
const template = spec?.template
if (!spec || !template) return preserved
const annotations = { ...(template.metadata?.annotations ?? {}) }
- delete annotations['clerum.io/credentials-revision']
+ for (const key of dropped) delete annotations[key]
return {
...preserved,
spec: {
@@ -4370,6 +4381,34 @@ function preserveChannelReaderDeploymentAnnotations(
}
}
+/**
+ * Host Deployment merge. `clerum.io/guardrails-revision` is controller-owned:
+ * dropping the whole `spec.guardrails` block omits it from the desired template,
+ * and that removal MUST reach the pod. mcp-host re-resolves guardrails live, but
+ * only while its Host watch is connected — the pod roll is what delivers the
+ * change to an agent whose watch has lapsed, so a removal that silently produced
+ * no diff left the agent enforcing guardrails the operator had uninstalled.
+ */
+function preserveHostDeploymentAnnotations(
+ desired: k8s.V1Deployment,
+ existing: k8s.V1Deployment
+): k8s.V1Deployment {
+ return preserveDeploymentAnnotationsExcept(desired, existing, [GUARDRAILS_REVISION_ANNOTATION])
+}
+
+/**
+ * Preserve operational annotations without retaining the channel-reader
+ * revision when HCC intentionally omits it from the desired pod template.
+ * That annotation is controller-owned and must be cleared when no backing
+ * CommunicationChannel Secret is resolvable.
+ */
+function preserveChannelReaderDeploymentAnnotations(
+ desired: k8s.V1Deployment,
+ existing: k8s.V1Deployment
+): k8s.V1Deployment {
+ return preserveDeploymentAnnotationsExcept(desired, existing, ['clerum.io/credentials-revision'])
+}
+
function normalizeDeploymentForComparison(deployment: k8s.V1Deployment): unknown {
const normalized = structuredClone(deployment)
delete normalized.status