Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(control-api): make host guardrail-hook ref writes atomic
Andy Paviaa234adf5caec
diff --git a/control-api/package-lock.json b/control-api/package-lock.jsonindex 305db4444..d6708a2cb 100644--- a/control-api/package-lock.json+++ b/control-api/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/image-policy": "file:../packages/image-policy",diff --git a/control-api/package.json b/control-api/package.jsonindex 5af12984c..0a440c422 100644--- a/control-api/package.json+++ b/control-api/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-api",- "version": "0.1.460",+ "version": "0.1.461","description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets","main": "dist/main.js","engines": {diff --git a/control-api/src/routes/admin/registry.ts b/control-api/src/routes/admin/registry.tsindex 6164b38f3..ce6d42c50 100644--- a/control-api/src/routes/admin/registry.ts+++ b/control-api/src/routes/admin/registry.ts@@ -11,7 +11,11 @@ import { K8sGateway } from '../../k8s.js'import type { UiAuthedRequest } from '../../middleware/controlUIAuth.js'import { rateLimitMiddleware } from '../../middleware/rateLimitMiddleware.js'import { type AdminUserRecord, findAdminById } from '../../services/adminAuthService.js'-import { listHostsReferencingHook, syncHookRefsInHosts } from '../../services/hostGuardrailRefs.js'+import {+ addHookRefToHost,+ listHostsReferencingHook,+ syncHookRefsInHosts,+} from '../../services/hostGuardrailRefs.js'import { createKey, listImages, listKeys, revokeKey } from '../../services/orgApiKeyClient.js'import type { RegistryEntry } from '../../services/registryClient.js'import {@@ -2039,27 +2043,21 @@ export function createAdminRegistryRouter(gateway?: K8sGateway): Router {// the CR (+ Secret) back rather than return a false success.const digest = image?.ref?.includes('@') ? image.ref.split('@')[1] : undefinedtry {- const freshHost = (await gateway.getResource(- 'hosts',- body.hostRef,- config.hostsNamespace- )) as HostShape- const g: HostGuardrailsShape = { ...(freshHost.spec?.guardrails ?? {}) }- const hooks: Record<string, Array<{ id: string; digest?: string }>> = {- ...(g.hooks ?? {}),- }- for (const phase of hookMeta.lifecyclePoints) {- const arr = hooks[phase] ? [...hooks[phase]] : []- if (!arr.some(h => h.id === crName)) {- arr.push({ id: crName, ...(digest ? { digest } : {}) })- }- hooks[phase] = arr- }- g.hooks = hooks- await gateway.updateResource(- 'hosts',+ // Atomic read→derive→write (shared with the uninstall/upgrade paths):+ // re-reads the Host, derives the hooks map from THAT read, and sends its+ // resourceVersion as the replace precondition. Deriving out here and+ // writing with a plain `updateResource` sends no precondition and replays+ // a stale map, so a concurrent install/uninstall on the same Host is+ // silently clobbered — leaving this CR installed but unreferenced, which+ // reads to the operator as an active guardrail that never runs+ // (fail-OPEN). Exhausted retries throw into the rollback below rather+ // than reporting a false success.+ await addHookRefToHost(+ gateway,body.hostRef,- { spec: { ...freshHost.spec, guardrails: g } },+ crName,+ hookMeta.lifecyclePoints,+ digest,config.hostsNamespace)} catch (err) {diff --git a/control-api/src/services/__tests__/hostGuardrailRefs.test.ts b/control-api/src/services/__tests__/hostGuardrailRefs.test.tsindex 9815b4967..82d15949e 100644--- a/control-api/src/services/__tests__/hostGuardrailRefs.test.ts+++ b/control-api/src/services/__tests__/hostGuardrailRefs.test.ts@@ -1,37 +1,71 @@import { describe, expect, it } from 'vitest'import type { K8sGateway } from '../../k8s.js'import {+ addHookRefToHost,listHostsReferencingHook,stripHookRefFromHosts,syncHookRefsInHosts,} from '../hostGuardrailRefs.js'+type HookRef = { id: string; digest?: string }type Host = {metadata: { name: string }- spec: { guardrails?: { hooks?: Record<string, Array<{ id: string; digest?: string }>> } }+ spec: { guardrails?: { hooks?: Record<string, HookRef[]> } }}-function gatewayWith(hosts: Host[]) {+/**+ * `onBeforeMutate` fires inside `mutateResource`, i.e. AFTER the caller's list+ * read and BEFORE the derive+write — the window a concurrent install/uninstall+ * commits in. `listResource` hands out clones so the caller cannot observe that+ * later write through its own snapshot, exactly like a real list.+ */+function gatewayWith(+ hosts: Host[],+ onBeforeMutate?: (store: Map<string, Host>, name: string) => void+) {const store = new Map<string, Host>(hosts.map(h => [h.metadata.name, structuredClone(h)]))const gateway = {- listResource: async (plural: string) => (plural === 'hosts' ? [...store.values()] : []),- updateResource: async (_p: string, name: string, body: { spec: Host['spec'] }) => {- const cur = store.get(name)!- store.set(name, { ...cur, spec: body.spec })+ listResource: async (plural: string) =>+ plural === 'hosts' ? structuredClone([...store.values()]) : [],+ mutateResource: async (+ _p: string,+ name: string,+ mutate: (current: {+ spec?: Record<string, unknown>+ }) => { spec: Record<string, unknown> } | null+ ) => {+ onBeforeMutate?.(store, name)+ const current = structuredClone(store.get(name)!)+ const next = await mutate(current)+ if (!next) return current+ store.set(name, { ...store.get(name)!, spec: next.spec as Host['spec'] })return store.get(name)},} as unknown as K8sGatewayreturn { gateway, store }}-const host = (- name: string,- hooks: Record<string, Array<{ id: string; digest?: string }>>-): Host => ({+const host = (name: string, hooks: Record<string, HookRef[]>): Host => ({metadata: { name },spec: { guardrails: { hooks } },})+/** A one-shot concurrent install: appends `hookId` to `preCall` on first call. */+function concurrentInstall(hookId: string) {+ let fired = false+ return (store: Map<string, Host>, name: string) => {+ if (fired) return+ fired = true+ const cur = store.get(name)!+ const hooks = { ...(cur.spec.guardrails?.hooks ?? {}) }+ hooks.preCall = [...(hooks.preCall ?? []), { id: hookId }]+ store.set(name, {+ ...cur,+ spec: { ...cur.spec, guardrails: { ...cur.spec.guardrails, hooks } },+ })+ }+}+describe('hostGuardrailRefs', () => {it('listHostsReferencingHook returns only Hosts that reference the id', async () => {const { gateway } = gatewayWith([@@ -43,6 +77,31 @@ describe('hostGuardrailRefs', () => {expect(hosts.map(h => h.metadata?.name)).toEqual(['a'])})+ it('addHookRefToHost references the hook in every declared phase', async () => {+ const { gateway, store } = gatewayWith([host('a', { preCall: [{ id: 'keep' }] })])+ await addHookRefToHost(+ gateway,+ 'a',+ 'hook-x',+ ['preCall', 'postCallSuccess'],+ 'sha256:NEW',+ 'mcp-host'+ )+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({+ preCall: [{ id: 'keep' }, { id: 'hook-x', digest: 'sha256:NEW' }],+ postCallSuccess: [{ id: 'hook-x', digest: 'sha256:NEW' }],+ })+ })++ it('addHookRefToHost is idempotent per phase', async () => {+ const { gateway, store } = gatewayWith([host('a', {})])+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')+ expect(store.get('a')!.spec.guardrails?.hooks?.preCall).toEqual([+ { id: 'hook-x', digest: 'sha256:NEW' },+ ])+ })+it('stripHookRefFromHosts removes the ref from every phase + host, keeping others', async () => {const { gateway, store } = gatewayWith([host('a', { preCall: [{ id: 'hook-x' }, { id: 'keep' }], moderate: [{ id: 'hook-x' }] }),@@ -78,4 +137,40 @@ describe('hostGuardrailRefs', () => {postCallSuccess: [{ id: 'hook-x', digest: 'sha256:NEW' }],})})++ // ── lost-update guards ────────────────────────────────────────────────────+ // Each write derives its map from a re-read taken INSIDE the write, so a ref+ // another operation committed after our read survives. Deriving from the+ // earlier snapshot and replacing the whole spec with no resourceVersion+ // precondition drops it silently — that hook stays installed but unreferenced,+ // i.e. an "active" guardrail that never runs.++ it('addHookRefToHost keeps a ref another install added first', async () => {+ const { gateway, store } = gatewayWith([host('a', {})], concurrentInstall('hook-z'))+ await addHookRefToHost(gateway, 'a', 'hook-x', ['preCall'], undefined, 'mcp-host')+ expect(store.get('a')!.spec.guardrails?.hooks?.preCall).toEqual([+ { id: 'hook-z' },+ { id: 'hook-x' },+ ])+ })++ it('stripHookRefFromHosts keeps a ref installed after the list read', async () => {+ const { gateway, store } = gatewayWith(+ [host('a', { preCall: [{ id: 'hook-x' }] })],+ concurrentInstall('hook-z')+ )+ await stripHookRefFromHosts(gateway, 'hook-x', 'mcp-host')+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({ preCall: [{ id: 'hook-z' }] })+ })++ it('syncHookRefsInHosts keeps a ref installed after the list read', async () => {+ const { gateway, store } = gatewayWith(+ [host('a', { preCall: [{ id: 'hook-x', digest: 'sha256:OLD' }] })],+ concurrentInstall('hook-z')+ )+ await syncHookRefsInHosts(gateway, 'hook-x', ['preCall'], 'sha256:NEW', 'mcp-host')+ expect(store.get('a')!.spec.guardrails?.hooks).toEqual({+ preCall: [{ id: 'hook-z' }, { id: 'hook-x', digest: 'sha256:NEW' }],+ })+ })})diff --git a/control-api/src/services/hostGuardrailRefs.ts b/control-api/src/services/hostGuardrailRefs.tsindex d441b9190..150a50476 100644--- a/control-api/src/services/hostGuardrailRefs.ts+++ b/control-api/src/services/hostGuardrailRefs.ts@@ -37,19 +37,33 @@ export async function listHostsReferencingHook(return (items ?? []).filter(h => referencesHook(h, hookId))}-async function writeHostHooks(+/**+ * Atomic read→derive→write of one Host's `guardrails.hooks` map.+ *+ * `mutateResource` re-reads the Host, runs `derive` against THAT read, and uses+ * its `resourceVersion` as the replace precondition (re-deriving on a 409).+ * Deriving INSIDE the callback is the whole point: the Hosts here come from a+ * single list read, so a map computed out here would be replayed against a Host+ * that a concurrent install/uninstall has since changed — a full-spec replace+ * with no precondition, i.e. a silent lost update. That leaves an LlmHook CR+ * installed but unreferenced, which reads as an active guardrail that never+ * runs (fail-OPEN).+ */+async function mutateHostHooks(gateway: K8sGateway,- host: HostResource,- hooks: Record<string, HookRef[]>,- hostsNamespace: string+ name: string,+ hostsNamespace: string,+ derive: (hooks: Record<string, HookRef[]>) => Record<string, HookRef[]>): Promise<void> {- const name = host.metadata?.name- if (!name) return- const guardrails = { ...(host.spec?.guardrails ?? {}), hooks }- await gateway.updateResource(+ await gateway.mutateResource('hosts',name,- { spec: { ...host.spec, guardrails } },+ current => {+ const spec = (current.spec ?? {}) as NonNullable<HostResource['spec']>+ const guardrails: HostGuardrails = { ...(spec.guardrails ?? {}) }+ guardrails.hooks = derive(guardrails.hooks ?? {})+ return { spec: { ...spec, guardrails } }+ },hostsNamespace)}@@ -65,6 +79,33 @@ function withoutHook(hooks: Record<string, HookRef[]>, hookId: string): Record<sreturn next}+/**+ * Referential integrity on install: reference `hookId` from every phase in+ * `lifecyclePoints` on ONE Host. Idempotent per phase (an existing ref is left+ * alone), and atomic — refs another operation added after this request started+ * are preserved rather than clobbered (see mutateHostHooks).+ */+export async function addHookRefToHost(+ gateway: K8sGateway,+ hostName: string,+ hookId: string,+ lifecyclePoints: string[],+ digest: string | undefined,+ hostsNamespace: string+): Promise<void> {+ await mutateHostHooks(gateway, hostName, hostsNamespace, current => {+ const hooks: Record<string, HookRef[]> = { ...current }+ for (const phase of lifecyclePoints) {+ const arr = hooks[phase] ? [...hooks[phase]] : []+ if (!arr.some(r => r.id === hookId)) {+ arr.push({ id: hookId, ...(digest ? { digest } : {}) })+ }+ hooks[phase] = arr+ }+ return hooks+ })+}+/*** Referential integrity on uninstall: remove every `{ id: hookId }` reference* from all referencing Hosts. Call this BEFORE deleting the LlmHook CR so a@@ -78,13 +119,10 @@ export async function stripHookRefFromHosts(const hosts = await listHostsReferencingHook(gateway, hookId, hostsNamespace)const touched: string[] = []for (const host of hosts) {- await writeHostHooks(- gateway,- host,- withoutHook(host.spec?.guardrails?.hooks ?? {}, hookId),- hostsNamespace- )- if (host.metadata?.name) touched.push(host.metadata.name)+ const name = host.metadata?.name+ if (!name) continue+ await mutateHostHooks(gateway, name, hostsNamespace, hooks => withoutHook(hooks, hookId))+ touched.push(name)}return touched}@@ -104,14 +142,18 @@ export async function syncHookRefsInHosts(const hosts = await listHostsReferencingHook(gateway, hookId, hostsNamespace)const touched: string[] = []for (const host of hosts) {- const hooks = withoutHook(host.spec?.guardrails?.hooks ?? {}, hookId)- for (const phase of lifecyclePoints) {- const arr = hooks[phase] ? [...hooks[phase]] : []- arr.push({ id: hookId, ...(digest ? { digest } : {}) })- hooks[phase] = arr- }- await writeHostHooks(gateway, host, hooks, hostsNamespace)- if (host.metadata?.name) touched.push(host.metadata.name)+ const name = host.metadata?.name+ if (!name) continue+ await mutateHostHooks(gateway, name, hostsNamespace, current => {+ const hooks = withoutHook(current, hookId)+ for (const phase of lifecyclePoints) {+ const arr = hooks[phase] ? [...hooks[phase]] : []+ arr.push({ id: hookId, ...(digest ? { digest } : {}) })+ hooks[phase] = arr+ }+ return hooks+ })+ touched.push(name)}return touched}