Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(control-api): make host guardrail-hook ref writes atomic

Andy Paviaa234adf5caec
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 305db4444..d6708a2cb 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/image-policy": "file:../packages/image-policy",
diff --git a/control-api/package.json b/control-api/package.json
index 5af12984c..0a440c422 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.460",
+ "version": "0.1.461",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.ts
index 6164b38f3..ce6d42c50 100644
--- a/control-api/src/routes/admin/registry.ts
+++ b/control-api/src/routes/admin/registry.ts
@@ -11,7 +11,11 @@ import { K8sGateway } from '../../k8s.js'
import type { UiAuthedRequest } from '../../middleware/controlUIAuth.js'
import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'
import { type AdminUserRecord, findAdminById } from '../../services/adminAuthService.js'
-import { listHostsReferencingHook, syncHookRefsInHosts } from '../../services/hostGuardrailRefs.js'
+import {
+ addHookRefToHost,
+ listHostsReferencingHook,
+ syncHookRefsInHosts,
+} from '../../services/hostGuardrailRefs.js'
import { createKey, listImages, listKeys, revokeKey } from '../../services/orgApiKeyClient.js'
import type { RegistryEntry } from '../../services/registryClient.js'
import {
@@ -2039,27 +2043,21 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {
// the CR (+ Secret) back rather than return a false success.
const digest = image?.ref?.includes('@') ? image.ref.split('@')[1] : undefined
try {
- const freshHost = (await gateway.getResource(
- 'hosts',
- body.hostRef,
- config.hostsNamespace
- )) as HostShape
- const g: HostGuardrailsShape = { ...(freshHost.spec?.guardrails ?? {}) }
- const hooks: Record<string, Array<{ id: string; digest?: string }>> = {
- ...(g.hooks ?? {}),
- }
- for (const phase of hookMeta.lifecyclePoints) {
- const arr = hooks[phase] ? [...hooks[phase]] : []
- if (!arr.some(h => h.id === crName)) {
- arr.push({ id: crName, ...(digest ? { digest } : {}) })
- }
- hooks[phase] = arr
- }
- g.hooks = hooks
- await gateway.updateResource(
- 'hosts',
+ // Atomic read→derive→write (shared with the uninstall/upgrade paths):
+ // re-reads the Host, derives the hooks map from THAT read, and sends its
+ // resourceVersion as the replace precondition. Deriving out here and
+ // writing with a plain `updateResource` sends no precondition and replays
+ // a stale map, so a concurrent install/uninstall on the same Host is
+ // silently clobbered — leaving this CR installed but unreferenced, which
+ // reads to the operator as an active guardrail that never runs
+ // (fail-OPEN). Exhausted retries throw into the rollback below rather
+ // than reporting a false success.
+ await addHookRefToHost(
+ gateway,
body.hostRef,
- { spec: { ...freshHost.spec, guardrails: g } },
+ crName,
+ hookMeta.lifecyclePoints,
+ digest,
config.hostsNamespace
)
} catch (err) {
diff --git a/control-api/src/services/__tests__/hostGuardrailRefs.test.ts b/control-api/src/services/__tests__/hostGuardrailRefs.test.ts
index 9815b4967..82d15949e 100644
--- a/control-api/src/services/__tests__/hostGuardrailRefs.test.ts
+++ b/control-api/src/services/__tests__/hostGuardrailRefs.test.ts
@@ -1,37 +1,71 @@
import { describe, expect, it } from 'vitest'
import type { K8sGateway } from '../../k8s.js'
import {
+ addHookRefToHost,
listHostsReferencingHook,
stripHookRefFromHosts,
syncHookRefsInHosts,
} from '../hostGuardrailRefs.js'
+type HookRef = { id: string; digest?: string }
type Host = {
metadata: { name: string }
- spec: { guardrails?: { hooks?: Record<string, Array<{ id: string; digest?: string }>> } }
+ spec: { guardrails?: { hooks?: Record<string, HookRef[]> } }
}
-function gatewayWith(hosts: Host[]) {
+/**
+ * `onBeforeMutate` fires inside `mutateResource`, i.e. AFTER the caller's list
+ * read and BEFORE the derive+write — the window a concurrent install/uninstall
+ * commits in. `listResource` hands out clones so the caller cannot observe that
+ * later write through its own snapshot, exactly like a real list.
+ */
+function gatewayWith(
+ hosts: Host[],
+ onBeforeMutate?: (store: Map<string, Host>, name: string) => void
+) {
const store = new Map<string, Host>(hosts.map(h => [h.metadata.name, structuredClone(h)]))
const gateway = {
- listResource: async (plural: string) => (plural === 'hosts' ? [...store.values()] : []),
- updateResource: async (_p: string, name: string, body: { spec: Host['spec'] }) => {
- const cur = store.get(name)!
- store.set(name, { ...cur, spec: body.spec })
+ listResource: async (plural: string) =>
+ plural === 'hosts' ? structuredClone([...store.values()]) : [],
+ mutateResource: async (
+ _p: string,
+ name: string,
+ mutate: (current: {
+ spec?: Record<string, unknown>
+ }) => { spec: Record<string, unknown> } | null
+ ) => {
+ onBeforeMutate?.(store, name)
+ const current = structuredClone(store.get(name)!)
+ const next = await mutate(current)
+ if (!next) return current
+ store.set(name, { ...store.get(name)!, spec: next.spec as Host['spec'] })
return store.get(name)
},
} as unknown as K8sGateway
return { gateway, store }
}
-const host = (
- name: string,
- hooks: Record<string, Array<{ id: string; digest?: string }>>
-): Host => ({
+const host = (name: string, hooks: Record<string, HookRef[]>): Host => ({
metadata: { name },
spec: { guardrails: { hooks } },
})
+/** A one-shot concurrent install: appends `hookId` to `preCall` on first call. */
+function concurrentInstall(hookId: string) {
+ let fired = false
+ return (store: Map<string, Host>, name: string) => {
+ if (fired) return
+ fired = true
+ const cur = store.get(name)!
+ const hooks = { ...(cur.spec.guardrails?.hooks ?? {}) }
+ hooks.preCall = [...(hooks.preCall ?? []), { id: hookId }]
+ store.set(name, {
+ ...cur,
+ spec: { ...cur.spec, guardrails: { ...cur.spec.guardrails, hooks } },
+ })
+ }
+}
+
describe('hostGuardrailRefs', () => {
it('listHostsReferencingHook returns only Hosts that reference the id', async () => {
const { gateway } = gatewayWith([
@@ -43,6 +77,31 @@ describe('hostGuardrailRefs', () => {
expect(hosts.map(h => h.metadata?.name)).toEqual(['a'])
})
+ it('addHookRefToHost references the hook in every declared phase', async () => {
+ const { gateway, store } = gatewayWith([host('a', { preCall: [{ id: 'keep' }] })])
+ await addHookRefToHost(
+ gateway,
+ 'a',
+ 'hook-x',
+ ['preCall', 'postCallSuccess'],
+ 'sha256:NEW',
+ 'mcp-host'
+ )
+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({
+ preCall: [{ id: 'keep' }, { id: 'hook-x', digest: 'sha256:NEW' }],
+ postCallSuccess: [{ id: 'hook-x', digest: 'sha256:NEW' }],
+ })
+ })
+
+ it('addHookRefToHost is idempotent per phase', async () => {
+ const { gateway, store } = gatewayWith([host('a', {})])
+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')
+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')
+ expect(store.get('a')!.spec.guardrails?.hooks?.preCall).toEqual([
+ { id: 'hook-x', digest: 'sha256:NEW' },
+ ])
+ })
+
it('stripHookRefFromHosts removes the ref from every phase + host, keeping others', async () => {
const { gateway, store } = gatewayWith([
host('a', { preCall: [{ id: 'hook-x' }, { id: 'keep' }], moderate: [{ id: 'hook-x' }] }),
@@ -78,4 +137,40 @@ describe('hostGuardrailRefs', () => {
postCallSuccess: [{ id: 'hook-x', digest: 'sha256:NEW' }],
})
})
+
+ // ── lost-update guards ────────────────────────────────────────────────────
+ // Each write derives its map from a re-read taken INSIDE the write, so a ref
+ // another operation committed after our read survives. Deriving from the
+ // earlier snapshot and replacing the whole spec with no resourceVersion
+ // precondition drops it silently — that hook stays installed but unreferenced,
+ // i.e. an "active" guardrail that never runs.
+
+ it('addHookRefToHost keeps a ref another install added first', async () => {
+ const { gateway, store } = gatewayWith([host('a', {})], concurrentInstall('hook-z'))
+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], undefined, 'mcp-host')
+ expect(store.get('a')!.spec.guardrails?.hooks?.preCall).toEqual([
+ { id: 'hook-z' },
+ { id: 'hook-x' },
+ ])
+ })
+
+ it('stripHookRefFromHosts keeps a ref installed after the list read', async () => {
+ const { gateway, store } = gatewayWith(
+ [host('a', { preCall: [{ id: 'hook-x' }] })],
+ concurrentInstall('hook-z')
+ )
+ await stripHookRefFromHosts(gateway, 'hook-x', 'mcp-host')
+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({ preCall: [{ id: 'hook-z' }] })
+ })
+
+ it('syncHookRefsInHosts keeps a ref installed after the list read', async () => {
+ const { gateway, store } = gatewayWith(
+ [host('a', { preCall: [{ id: 'hook-x', digest: 'sha256:OLD' }] })],
+ concurrentInstall('hook-z')
+ )
+ await syncHookRefsInHosts(gateway, 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')
+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({
+ preCall: [{ id: 'hook-z' }, { id: 'hook-x', digest: 'sha256:NEW' }],
+ })
+ })
})
diff --git a/control-api/src/services/hostGuardrailRefs.ts b/control-api/src/services/hostGuardrailRefs.ts
index d441b9190..150a50476 100644
--- a/control-api/src/services/hostGuardrailRefs.ts
+++ b/control-api/src/services/hostGuardrailRefs.ts
@@ -37,19 +37,33 @@ export async function listHostsReferencingHook(
return (items ?? []).filter(h => referencesHook(h, hookId))
}
-async function writeHostHooks(
+/**
+ * Atomic read→derive→write of one Host's `guardrails.hooks` map.
+ *
+ * `mutateResource` re-reads the Host, runs `derive` against THAT read, and uses
+ * its `resourceVersion` as the replace precondition (re-deriving on a 409).
+ * Deriving INSIDE the callback is the whole point: the Hosts here come from a
+ * single list read, so a map computed out here would be replayed against a Host
+ * that a concurrent install/uninstall has since changed — a full-spec replace
+ * with no precondition, i.e. a silent lost update. That leaves an LlmHook CR
+ * installed but unreferenced, which reads as an active guardrail that never
+ * runs (fail-OPEN).
+ */
+async function mutateHostHooks(
gateway: K8sGateway,
- host: HostResource,
- hooks: Record<string, HookRef[]>,
- hostsNamespace: string
+ name: string,
+ hostsNamespace: string,
+ derive: (hooks: Record<string, HookRef[]>) => Record<string, HookRef[]>
): Promise<void> {
- const name = host.metadata?.name
- if (!name) return
- const guardrails = { ...(host.spec?.guardrails ?? {}), hooks }
- await gateway.updateResource(
+ await gateway.mutateResource(
'hosts',
name,
- { spec: { ...host.spec, guardrails } },
+ current => {
+ const spec = (current.spec ?? {}) as NonNullable<HostResource['spec']>
+ const guardrails: HostGuardrails = { ...(spec.guardrails ?? {}) }
+ guardrails.hooks = derive(guardrails.hooks ?? {})
+ return { spec: { ...spec, guardrails } }
+ },
hostsNamespace
)
}
@@ -65,6 +79,33 @@ function withoutHook(hooks: Record<string, HookRef[]>, hookId: string): Record<s
return next
}
+/**
+ * Referential integrity on install: reference `hookId` from every phase in
+ * `lifecyclePoints` on ONE Host. Idempotent per phase (an existing ref is left
+ * alone), and atomic — refs another operation added after this request started
+ * are preserved rather than clobbered (see mutateHostHooks).
+ */
+export async function addHookRefToHost(
+ gateway: K8sGateway,
+ hostName: string,
+ hookId: string,
+ lifecyclePoints: string[],
+ digest: string | undefined,
+ hostsNamespace: string
+): Promise<void> {
+ await mutateHostHooks(gateway, hostName, hostsNamespace, current => {
+ const hooks: Record<string, HookRef[]> = { ...current }
+ for (const phase of lifecyclePoints) {
+ const arr = hooks[phase] ? [...hooks[phase]] : []
+ if (!arr.some(r => r.id === hookId)) {
+ arr.push({ id: hookId, ...(digest ? { digest } : {}) })
+ }
+ hooks[phase] = arr
+ }
+ return hooks
+ })
+}
+
/**
* Referential integrity on uninstall: remove every `{ id: hookId }` reference
* from all referencing Hosts. Call this BEFORE deleting the LlmHook CR so a
@@ -78,13 +119,10 @@ export async function stripHookRefFromHosts(
const hosts = await listHostsReferencingHook(gateway, hookId, hostsNamespace)
const touched: string[] = []
for (const host of hosts) {
- await writeHostHooks(
- gateway,
- host,
- withoutHook(host.spec?.guardrails?.hooks ?? {}, hookId),
- hostsNamespace
- )
- if (host.metadata?.name) touched.push(host.metadata.name)
+ const name = host.metadata?.name
+ if (!name) continue
+ await mutateHostHooks(gateway, name, hostsNamespace, hooks => withoutHook(hooks, hookId))
+ touched.push(name)
}
return touched
}
@@ -104,14 +142,18 @@ export async function syncHookRefsInHosts(
const hosts = await listHostsReferencingHook(gateway, hookId, hostsNamespace)
const touched: string[] = []
for (const host of hosts) {
- const hooks = withoutHook(host.spec?.guardrails?.hooks ?? {}, hookId)
- for (const phase of lifecyclePoints) {
- const arr = hooks[phase] ? [...hooks[phase]] : []
- arr.push({ id: hookId, ...(digest ? { digest } : {}) })
- hooks[phase] = arr
- }
- await writeHostHooks(gateway, host, hooks, hostsNamespace)
- if (host.metadata?.name) touched.push(host.metadata.name)
+ const name = host.metadata?.name
+ if (!name) continue
+ await mutateHostHooks(gateway, name, hostsNamespace, current => {
+ const hooks = withoutHook(current, hookId)
+ for (const phase of lifecyclePoints) {
+ const arr = hooks[phase] ? [...hooks[phase]] : []
+ arr.push({ id: hookId, ...(digest ? { digest } : {}) })
+ hooks[phase] = arr
+ }
+ return hooks
+ })
+ touched.push(name)
}
return touched
}