Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(#408): generation-aware network-ready handshake (WRC <-> HCC)
Alfredo Lopez568a3a044b44
diff --git a/docs/architecture/platform-topology.md b/docs/architecture/platform-topology.mdindex d120a5739..9fcbb32ff 100644--- a/docs/architecture/platform-topology.md+++ b/docs/architecture/platform-topology.md@@ -840,7 +840,7 @@ Three approaches were evaluated:| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- || **A — ReadinessGate** | HCC patches `pod.status.conditions` with `clerum.io/network-isolated: True` after applying NPs | Standard K8s pattern, Service won't route until ready | HCC must watch pods in all namespaces, direct pod patching violates Invariant #1 (CRD-only communication), adds Pod RBAC to HCC, complex lifecycle (pod restart = re-patch) || **B — Init container** | Sidecar init container blocks until NetworkPolicy exists | No HCC changes needed | Requires shared ServiceAccount, polling from within pod is fragile, no guarantee NP is _applied_ vs just _created_ |-| **C — Pre-deploy annotation handshake** | WRC creates McpServer CRD with `clerum.io/pre-deploy: true` → HCC applies NPs → HCC sets `clerum.io/network-ready: true` → WRC creates Deployment | 100% CRD-based (Invariant #1 preserved), no pod patching, no RBAC expansion, reuses existing watch loops | Adds ~1-5s latency to first deployment, requires timeout handling for HCC unavailability |+| **C — Pre-deploy annotation handshake** | WRC creates McpServer CRD with `clerum.io/pre-deploy: true` → HCC applies NPs → HCC sets `clerum.io/network-ready: true` (+ `network-ready-observed-generation`, Issue #408) → WRC creates Deployment | 100% CRD-based (Invariant #1 preserved), no pod patching, no RBAC expansion, reuses existing watch loops | Adds ~1-5s latency to first deployment, requires timeout handling for HCC unavailability |**Decision**: **Option C** — the annotation handshake preserves the CRD-only communication invariant while closing the vulnerability window. ReadinessGate (Option A) is deferred indefinitely as Option C provides equivalent security guarantees with simpler implementation.@@ -857,7 +857,7 @@ WRC McpServer CRD HC│ ├── watch triggers ───────────────►││ │ ││ │◄── HCC applies L2/L3 NPs ───────┤- │ │ + sets annotation: │+ │ │ + sets ack + generation: ││ │ clerum.io/network-ready: true ││ │ ││◄── Step 7c: Poll for network-ready ──────┤ │@@ -872,10 +872,11 @@ WRC McpServer CRD HC#### 11.4.3 Annotations-| Annotation | Set by | Value | Meaning |-| ------------------------- | ------ | -------- | ---------------------------------------------------------------------------------- |-| `clerum.io/pre-deploy` | WRC | `"true"` | McpServer CRD created before workload pods exist; HCC should apply NPs proactively |-| `clerum.io/network-ready` | HCC | `"true"` | L2/L3 NetworkPolicies applied; safe to start workload pods |+| Annotation | Set by | Value | Meaning |+| -------------------------------------------- | ------ | ---------------------- | ------------------------------------------------------------------------------------------------------------------- |+| `clerum.io/pre-deploy` | WRC | `"true"` | McpServer CRD created before workload pods exist; HCC should apply NPs proactively |+| `clerum.io/network-ready` | HCC | `"true"` | L2/L3 NetworkPolicies applied; safe to start workload pods. Valid only together with a matching generation stamp |+| `clerum.io/network-ready-observed-generation` | HCC | `"<metadata.generation>"` | Generation HCC observed when acking (Issue #408). WRC honors `network-ready` only when this equals the current generation, so a stale ack from a prior generation cannot satisfy the gate. HCC re-acks (re-stamps) when the generation advances |#### 11.4.4 Timeout and Graceful Degradationdiff --git a/host-context-controller/package-lock.json b/host-context-controller/package-lock.jsonindex e9b28c157..6b6a904d8 100644--- a/host-context-controller/package-lock.json+++ b/host-context-controller/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.171","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.171","license": "SEE LICENSE IN LICENSE","dependencies": {"@clerum/image-policy": "file:../packages/image-policy",diff --git a/host-context-controller/package.json b/host-context-controller/package.jsonindex 17c98b81a..bf06f1a88 100644--- a/host-context-controller/package.json+++ b/host-context-controller/package.json@@ -1,6 +1,6 @@{"name": "clerum-host-context-controller",- "version": "0.1.170",+ "version": "0.1.171","description": "Host Context Controller - provides McpServer CRDs to mcp-host via REST API","main": "dist/main.js","engines": {diff --git a/host-context-controller/src/reconciler.test.ts b/host-context-controller/src/reconciler.test.tsindex 92e74d6e8..349a2a714 100644--- a/host-context-controller/src/reconciler.test.ts+++ b/host-context-controller/src/reconciler.test.ts@@ -149,6 +149,89 @@ describe('Reconciler managed:false guard (Risk 1.7)', () => {)})+ it('Issue #408: stamps the observed generation alongside the network-ready ack', async () => {+ const server = {+ ...makeServer({ name: 'stdio-mcp', managed: true }),+ generation: 3,+ annotations: { 'clerum.io/pre-deploy': 'true' },+ }++ await reconciler.reconcile(server)++ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({+ plural: 'mcpservers',+ name: 'stdio-mcp',+ body: {+ metadata: {+ annotations: {+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '3',+ },+ },+ },+ }),+ expect.objectContaining({ middleware: expect.any(Array) })+ )+ })++ it('Issue #408: re-acks when the stamped generation is stale (spec changed since last ack)', async () => {+ const server = {+ ...makeServer({ name: 'stdio-mcp', managed: true }),+ generation: 3,+ annotations: {+ 'clerum.io/pre-deploy': 'true',+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '2',+ },+ }++ await reconciler.reconcile(server)++ // Old guard (network-ready !== 'true') would skip the patch; the fix must re-ack+ // for the current generation, re-stamping observed-generation to '3'.+ expect(customApi.patchNamespacedCustomObject).toHaveBeenCalledWith(+ expect.objectContaining({+ plural: 'mcpservers',+ name: 'stdio-mcp',+ body: {+ metadata: {+ annotations: {+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '3',+ },+ },+ },+ }),+ expect.objectContaining({ middleware: expect.any(Array) })+ )+ })++ it('Issue #408: does NOT re-ack when the stamped generation already matches (no write storm)', async () => {+ const server = {+ ...makeServer({ name: 'stdio-mcp', managed: true }),+ generation: 3,+ annotations: {+ 'clerum.io/pre-deploy': 'true',+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '3',+ },+ }++ await reconciler.reconcile(server)++ expect(customApi.patchNamespacedCustomObject).not.toHaveBeenCalledWith(+ expect.objectContaining({+ body: expect.objectContaining({+ metadata: expect.objectContaining({+ annotations: expect.objectContaining({ 'clerum.io/network-ready': 'true' }),+ }),+ }),+ }),+ expect.anything()+ )+ })+it('should create Deployment when managed: undefined (default: true)', async () => {const server = makeServer({ name: 'mongo-mcp' })await reconciler.reconcile(server)diff --git a/host-context-controller/src/reconciler.ts b/host-context-controller/src/reconciler.tsindex caba6be91..9908c347b 100644--- a/host-context-controller/src/reconciler.ts+++ b/host-context-controller/src/reconciler.ts@@ -74,6 +74,9 @@ type McpServerReconcilerDeps = {// G2: Pre-deploy handshake constantsconst PRE_DEPLOY_ANNOTATION = 'clerum.io/pre-deploy'const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'+// Issue #408: stamp the generation observed when acking so WRC can reject a stale+// ack carried over from a previous generation.+const NETWORK_READY_GENERATION_ANNOTATION = 'clerum.io/network-ready-observed-generation'// Issue #223: sha256 of the envSecret content, carried on the POD TEMPLATE (not// the Deployment object) so that changing it changes the pod template hash and@@ -1499,6 +1502,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolvmetadata: {annotations: {[NETWORK_READY_ANNOTATION]: 'true',+ // Issue #408: stamp the acked generation so WRC's waitForNetworkReady+ // can reject a stale ack from a previous generation.+ ...(typeof server.generation === 'number'+ ? { [NETWORK_READY_GENERATION_ANNOTATION]: String(server.generation) }+ : {}),},},},@@ -1930,7 +1938,11 @@ ${authHeaderLines ? '\n # ── Credential auth headers (envsubst-resolv// NetworkPolicies are in place and workload can proceed.if (server.annotations?.[PRE_DEPLOY_ANNOTATION] === 'true' &&- server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true'+ (server.annotations?.[NETWORK_READY_ANNOTATION] !== 'true' ||+ // Issue #408: re-ack when the stamped generation is stale (spec changed since+ // the last ack). A non-numeric generation keeps the legacy behavior.+ (typeof server.generation === 'number' &&+ server.annotations?.[NETWORK_READY_GENERATION_ANNOTATION] !== String(server.generation)))) {try {await this.setNetworkReadyAnnotation(server)diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 3de2ef679..747a67f9e 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.294",+ "version": "0.1.295","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.294",+ "version": "0.1.295","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 04cf4a1e9..c775934de 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.294",+ "version": "0.1.295","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/mcpDelegation.test.ts b/workflow-recipes/src/reconciler/mcpDelegation.test.tsindex 87997894a..929ff5419 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.test.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.test.ts@@ -18,6 +18,7 @@ import {preDeployMcpServers,transportWorkloadSecretDenied,waitForExternalEgressReady,+ waitForNetworkReady,} from './mcpDelegation'import type { SecretAccess } from './resourceBuilder'import { privateWorkflowContextName } from './workflowContext'@@ -1347,6 +1348,70 @@ describe('delegateTransportWorkloads', () => {expect(mcpServerReplaceCalls.length).toBeGreaterThanOrEqual(1)})+ it('Issue #408: drops the carried-over network-ready ack pair from the replace on a spec change', async () => {+ const MCPSERVER_PLURAL = 'mcpservers'+ const SPEC_HASH = 'clerum.io/spec-hash'+ const NETWORK_READY = 'clerum.io/network-ready'+ const NETWORK_READY_GEN = 'clerum.io/network-ready-observed-generation'+ const recipe = makeRecipe()+ let storedMcpServer:+ | { metadata?: { annotations?: Record<string, string> }; spec?: Record<string, unknown> }+ | undefined++ const mockCustomApi = {+ getNamespacedCustomObject: vi.fn().mockImplementation((args: { plural: string }) => {+ if (args.plural === MCPSERVER_PLURAL && storedMcpServer) {+ return Promise.resolve(storedMcpServer)+ }+ return Promise.reject({ code: 404 })+ }),+ createNamespacedCustomObject: vi+ .fn()+ .mockImplementation((args: { plural: string; body: unknown }) => {+ if (args.plural === MCPSERVER_PLURAL) {+ storedMcpServer = args.body as typeof storedMcpServer+ }+ return Promise.resolve({})+ }),+ replaceNamespacedCustomObject: vi.fn().mockResolvedValue({}),+ }+ const mockCoreApi = { createNamespacedService: vi.fn().mockResolvedValue({}) }+ const deps: DelegationDeps = {+ customApi: mockCustomApi as unknown as DelegationDeps['customApi'],+ coreApi: mockCoreApi as unknown as DelegationDeps['coreApi'],+ }++ await delegateTransportWorkloads(deps, recipe, 'mcp-server', new Map())+ expect(storedMcpServer).toBeDefined()++ // Simulate a prior-generation HCC ack carried on the live object, plus a stale+ // spec-hash so the desired manifest differs and the replace path is taken.+ storedMcpServer!.metadata = {+ ...(storedMcpServer!.metadata ?? {}),+ annotations: {+ ...(storedMcpServer!.metadata?.annotations ?? {}),+ [SPEC_HASH]: 'stale-hash',+ [NETWORK_READY]: 'true',+ [NETWORK_READY_GEN]: '1',+ },+ }+ mockCustomApi.replaceNamespacedCustomObject.mockClear()+ await delegateTransportWorkloads(deps, recipe, 'mcp-server', new Map())++ const replaceCall = mockCustomApi.replaceNamespacedCustomObject.mock.calls.find(+ (call: unknown[]) => (call[0] as { plural?: string } | undefined)?.plural === MCPSERVER_PLURAL+ ) as [{ body: { metadata: { annotations: Record<string, string> } } }] | undefined+ expect(replaceCall).toBeDefined()++ const replacedAnnotations = replaceCall![0].body.metadata.annotations+ // The stale network ack pair must NOT be carried forward into the new generation.+ expect(replacedAnnotations[NETWORK_READY]).toBeUndefined()+ expect(replacedAnnotations[NETWORK_READY_GEN]).toBeUndefined()+ // Non-network annotations (the freshly stamped spec-hash from the manifest) survive.+ expect(replacedAnnotations[SPEC_HASH]).toBeDefined()+ expect(replacedAnnotations[SPEC_HASH]).not.toBe('stale-hash')+ })+it('does NOT patch shared Context (H-04 isolation — no patchNamespacedCustomObject)', async () => {const mockCustomApi = {createNamespacedCustomObject: vi.fn().mockResolvedValue({}),@@ -1999,6 +2064,106 @@ describe('waitForExternalEgressReady', () => {})})+// ─── Issue #408 — waitForNetworkReady must be generation-aware ─────────────────+// The flat `clerum.io/network-ready: "true"` ack carries no generation. A stale+// ack carried over from a previous generation must NOT satisfy the gate for the+// current generation. Mirrors waitForExternalEgressReady's observedGeneration+// freshness check, but for the annotation pair.+describe('waitForNetworkReady (Issue #408 generation-aware gate)', () => {+ it('rejects a network-ready ack stamped for an older generation', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: {+ generation: 2,+ annotations: {+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '1',+ },+ },+ }),+ } as any,+ coreApi: {} as any,+ }++ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)++ expect(result.ready).toBe(false)+ expect(result.pending).toEqual(['srv-a'])+ })++ it('rejects a network-ready ack that lacks the generation stamp', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: {+ generation: 2,+ annotations: { 'clerum.io/network-ready': 'true' },+ },+ }),+ } as any,+ coreApi: {} as any,+ }++ const result = await waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)++ expect(result.ready).toBe(false)+ expect(result.pending).toEqual(['srv-a'])+ })++ it('accepts a network-ready ack stamped for the current generation', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: {+ generation: 2,+ annotations: {+ 'clerum.io/network-ready': 'true',+ 'clerum.io/network-ready-observed-generation': '2',+ },+ },+ }),+ } as any,+ coreApi: {} as any,+ }++ await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ ready: true,+ pending: [],+ })+ })++ it('tolerates a missing (non-numeric) generation and accepts the flat ack', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockResolvedValue({+ metadata: { annotations: { 'clerum.io/network-ready': 'true' } },+ }),+ } as any,+ coreApi: {} as any,+ }++ await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ ready: true,+ pending: [],+ })+ })++ it('treats a 404 (McpServer deleted) as resolved', async () => {+ const deps: DelegationDeps = {+ customApi: {+ getNamespacedCustomObject: vi.fn().mockRejectedValue({ code: 404 }),+ } as any,+ coreApi: {} as any,+ }++ await expect(waitForNetworkReady(deps, ['srv-a'], 'mcp-server', 25)).resolves.toEqual({+ ready: true,+ pending: [],+ })+ })+})+// ─── Issue #637 — transport Secret ownership gate ──────────────────────────────// Proves the transport delegation path (McpServer CRD copy) is fail-closed for// cross-recipe Secret ownership, mirroring the non-transport render gate. HCCdiff --git a/workflow-recipes/src/reconciler/mcpDelegation.ts b/workflow-recipes/src/reconciler/mcpDelegation.tsindex 06a65e8c5..df37c46ef 100644--- a/workflow-recipes/src/reconciler/mcpDelegation.ts+++ b/workflow-recipes/src/reconciler/mcpDelegation.ts@@ -439,9 +439,18 @@ async function ensureMcpServer(}// Recipe-owned: safe to replace (idempotent update path).- // Merge annotations: existing annotations (e.g. HCC's "network-ready: true") are preserved- // as the base; WRC manifest annotations override only the keys they specify.- // This prevents WRC updates from erasing HCC-managed annotations during concurrent reconciles.+ // Merge annotations: existing annotations are preserved as the base; WRC manifest+ // annotations override only the keys they specify. This prevents WRC updates from+ // erasing HCC-managed annotations during concurrent reconciles.+ // EXCEPTION (Issue #408): the pre-deploy network ack pair+ // (clerum.io/network-ready + clerum.io/network-ready-observed-generation) is dropped+ // from the carry-over here. This branch runs only when the DESIRED manifest's+ // spec-hash differs from the stored one (the idempotency gate below returns false and+ // skips the replace when they match), so a byte-identical desired object is never+ // rewritten by this path. Dropping the ack is idempotent: HCC re-acks after+ // re-applying NetworkPolicies for the current generation via its+ // (annotation !== 'true' || stamped-generation !== current) guard, so a stale ack+ // cannot satisfy the gate for the new generation.const rv = existing.metadata?.resourceVersionconst existingAnnotations = existing.metadata?.annotations ?? {}const manifestAnnotations =@@ -465,7 +474,10 @@ async function ensureMcpServer(return false}- const mergedAnnotations = { ...existingAnnotations, ...manifestAnnotations }+ const carriedAnnotations: Record<string, string> = { ...existingAnnotations }+ delete carriedAnnotations[NETWORK_READY_ANNOTATION]+ delete carriedAnnotations[NETWORK_READY_GENERATION_ANNOTATION]+ const mergedAnnotations = { ...carriedAnnotations, ...manifestAnnotations }const updatedManifest = {...manifest,metadata: {@@ -773,6 +785,9 @@ export async function ensureRecipeContext(*/export const PRE_DEPLOY_ANNOTATION = 'clerum.io/pre-deploy'export const NETWORK_READY_ANNOTATION = 'clerum.io/network-ready'+// Issue #408: HCC stamps the McpServer generation it observed when acking network+// readiness, so WRC can reject a stale ack carried over from a prior generation.+export const NETWORK_READY_GENERATION_ANNOTATION = 'clerum.io/network-ready-observed-generation'const NETWORK_READY_POLL_INTERVAL_MS = 1_000const NETWORK_READY_TIMEOUT_MS = 30_000@@ -916,10 +931,21 @@ export async function waitForNetworkReady(namespace,plural: MCPSERVER_PLURAL,name,- })) as { metadata?: { annotations?: Record<string, string> } }+ })) as {+ metadata?: { generation?: number; annotations?: Record<string, string> }+ }+ // Issue #408: mirror waitForExternalEgressReady's freshness check. The flat+ // 'true' ack carries no generation, so trust it only when HCC's stamped+ // generation matches the current one. A non-numeric generation is tolerated+ // as fresh (same tolerance as the condition-based gate).+ const annotations = mcpServer.metadata?.annotations+ const currentGeneration = mcpServer.metadata?.generation+ const ackFresh =+ typeof currentGeneration !== 'number' ||+ annotations?.[NETWORK_READY_GENERATION_ANNOTATION] === String(currentGeneration)return {name,- ready: mcpServer.metadata?.annotations?.[NETWORK_READY_ANNOTATION] === 'true',+ ready: annotations?.[NETWORK_READY_ANNOTATION] === 'true' && ackFresh,}} catch (error) {// 404 = McpServer deleted externally — treat as resolved