Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): apply NetworkPolicies through the spec-hash gate (#575)
Alfredo Lopez11db811887ee
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.mdindex 910a6999d..97bd36ae3 100644--- a/docs/deploy/workflow-recipes-guide.md+++ b/docs/deploy/workflow-recipes-guide.md@@ -302,7 +302,10 @@ kubectl delete workflowrecipe my-nginx -n sandbox-recipes## Reconcile idempotency & drift behaviorWRC reconciles every recipe on a periodic resync. Each managed object (Deployment,-StatefulSet, headless Service, CronJob, Job, DaemonSet, and the McpServer CRD) is+StatefulSet, headless Service, CronJob, Job, DaemonSet, the McpServer CRD, and every+WRC-authored NetworkPolicy applied through `applyNetworkPolicy` — `ui-ingress-*`,+`wl-ingress-*`, `wr-intdep-*`, `wf-*-oauth-broker-egress`, the three webhook-gateway+policies, and `wl-egress-*` once issue #299 has decided a write is due) isapplied through a **spec-hash idempotency gate**: WRC stamps a `clerum.io/spec-hash`annotation (a hash of the _desired_ manifest) and, on the already-exists path, **skipsthe write entirely when the existing object's stamped hash matches**. This keepsdiff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex bd7095d74..fbee3b101 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.339",+ "version": "0.1.340","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.339",+ "version": "0.1.340","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex c043e5cb5..7747bce78 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.339",+ "version": "0.1.340","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 1018b40a9..cfccea8ec 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -18,6 +18,7 @@ import {resolveScopedWorkloadResourceName,resolveWorkloadRuntimeResourceName,} from './resourceBuilder'+import { stampSpecHash } from './specHash'import {TRANSIENT_REQUEUE_BASE_MS,WORKFLOW_PROGRESS_REQUEUE_BASE_MS,@@ -769,6 +770,212 @@ describe('WorkflowRecipeReconciler', () => {])})+ // ─── issue #575: the spec-hash gate applyNetworkPolicy never opted into ──────+ //+ // `createOrReplace` takes a fourth `idempotency` argument that stamps the+ // desired manifest with a SHA-256 of its content and compares it against the+ // annotation on the live object before replacing. Five callers pass it —+ // Deployment, headless Service, CronJob, Job, DaemonSet. `applyNetworkPolicy`+ // did not, so every WRC NetworkPolicy funnelling through it was rewritten on+ // every reconcile pass without comparing anything: ~12 200 PUTs/hour of objects+ // the apiserver then discarded as unchanged.+ //+ // The ownership assertion is why this is not a one-line change. It runs inside+ // the replace path today; a gate that returns first would skip it, so it moves+ // into `readExisting` — the very read the gate evaluates — and the result is+ // cached only AFTER it passes, so the replace path can never inherit a vetoed+ // object.+ describe('applyNetworkPolicy spec-hash gate (issue #575)', () => {+ const SPEC_HASH = 'clerum.io/spec-hash'+ const NS = 'sandbox-recipes'+ const clone = <T>(v: T): T => JSON.parse(JSON.stringify(v)) as T+ type Priv = { applyNetworkPolicy: (p: k8s.V1NetworkPolicy, ns: string) => Promise<void> }+ const apply = (p: k8s.V1NetworkPolicy) =>+ (reconciler as unknown as Priv).applyNetworkPolicy(p, NS)++ const plainPolicy = (): k8s.V1NetworkPolicy => ({+ apiVersion: 'networking.k8s.io/v1',+ kind: 'NetworkPolicy',+ metadata: {+ name: 'wl-ingress-r-w',+ namespace: NS,+ labels: { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': 'r' },+ },+ spec: {+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },+ policyTypes: ['Ingress'],+ ingress: [{ ports: [{ port: 8080, protocol: 'TCP' }] }],+ },+ })++ const intdepPolicy = (): k8s.V1NetworkPolicy => ({+ ...plainPolicy(),+ metadata: {+ name: 'wr-intdep-egress-r-w',+ namespace: NS,+ labels: {+ 'clerum.io/managed-by': 'workflow-recipes',+ 'clerum.io/recipe': 'r',+ 'clerum.io/policy-type': 'internal-dependency',+ },+ },+ })++ const ownedExisting = (extra: Record<string, unknown> = {}) => ({+ metadata: {+ name: 'wl-ingress-r-w',+ resourceVersion: '9',+ labels: plainPolicy().metadata!.labels,+ ...extra,+ },+ })++ it('T1 does NOT replace a NetworkPolicy whose spec-hash is unchanged, and says so', async () => {+ await apply(plainPolicy())+ const sealed = clone(+ (+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.at(-1)![0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ )+ expect(sealed.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()++ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({+ ...sealed,+ metadata: { ...sealed.metadata, resourceVersion: '9' },+ })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)+ try {+ await apply(plainPolicy())+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ // The skip line is load-bearing, not decoration: it is what keeps the+ // ~16s reconcile cadence visible once the writes stop. A silent gate+ // would make the symptom of #460 invisible.+ expect(logSpy.mock.calls.flat().join(' ')).toContain(+ 'NetworkPolicy "wl-ingress-r-w" in sandbox-recipes unchanged (spec-hash match); skipping update'+ )+ } finally {+ logSpy.mockRestore()+ }+ })++ it('T2 replaces when the spec-hash differs, reading once and using the live resourceVersion', async () => {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(+ ownedExisting({ annotations: { [SPEC_HASH]: 'stale' } })+ )++ await apply(plainPolicy())++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ expect(body.metadata?.resourceVersion).toBe('9')+ expect(body.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()+ // One read serves both the gate and the replace, so the replace cannot act+ // on a different resourceVersion than the one the gate evaluated.+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })++ it('T3 writes once to seal a pre-existing policy with no spec-hash, then skips', async () => {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(ownedExisting())++ await apply(plainPolicy())+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const sealed = clone(+ (+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {+ body: k8s.V1NetworkPolicy+ }+ ).body+ )+ expect(sealed.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()++ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(sealed)++ await apply(plainPolicy())+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })++ it('T4 does not let a matching spec-hash shortcut the ownership assertion', async () => {+ const desired = intdepPolicy()+ const sameHash = stampSpecHash(clone(desired))+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ // Foreign owner, carrying a hash the gate would happily match on.+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({+ metadata: {+ name: 'wr-intdep-egress-r-w',+ resourceVersion: '9',+ annotations: { [SPEC_HASH]: sameHash },+ labels: { 'clerum.io/managed-by': 'hcc' },+ },+ })++ await expect(apply(desired)).rejects.toThrow(+ /Refusing to replace NetworkPolicy "wr-intdep-egress-r-w"/+ )+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ // Two reads: the gate's read raises the veto, `applyIsNoop` treats any+ // throw as "cannot prove this is a no-op" and falls through, and the replace+ // path re-reads and re-asserts, where the veto propagates. The write never+ // happens either way. What this test rules out is the assertion living ONLY+ // in the replace path: there, a matching hash would make the gate skip and+ // this call would resolve instead of rejecting.+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })++ it('T5 falls through to the replace when the gate read fails with an HTTP status', async () => {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 500 })+ .mockResolvedValueOnce(ownedExisting())++ await apply(plainPolicy())++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })++ it('T5c falls through on a codeless read error, the shape `:1354` pins for Deployment', async () => {+ // The rule cannot key on the shape of the error. A bare Error with no code+ // is a legitimate read failure — the existing Deployment test injects+ // exactly `new Error('transient read timeout')` and asserts the write still+ // happens — so `applyIsNoop` swallows everything and falls open. A version+ // of this gate that classified errors instead broke that test; this pins the+ // behaviour for NetworkPolicy so the same idea cannot come back quietly.+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce(new Error('transient read timeout'))+ .mockResolvedValueOnce(ownedExisting())++ await apply(plainPolicy())++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })++ it('T5b falls through to the replace on a retryable transport error', async () => {+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy+ .mockRejectedValueOnce({ code: 'ECONNRESET' })+ .mockResolvedValueOnce(ownedExisting())++ await apply(plainPolicy())++ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)+ })+ })+describe('idempotent apply (stops generation churn)', () => {const SPEC_HASH = 'clerum.io/spec-hash'const clone = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T@@ -9191,6 +9398,47 @@ describe('WorkflowRecipeReconciler', () => {expect(allFalse).toBe(true)})+ it('T6 (#575) applies the 3 gateway NetworkPolicies through the spec-hash gate', async () => {+ // Two jobs. First: the gateway policies used to be written by an inline+ // create-or-replace with no gate, so they were three no-op PUTs per pass.+ // Second, and the reason this test is here rather than asserted by+ // inspection: it proves the gateway builder is DETERMINISTIC. A gate only+ // closes if the manifest hashes the same on the next pass, and a builder+ // that iterated a Set or stamped a timestamp would leave the gate open+ // forever — the failure mode of #413 and #579. If this test goes red, the+ // builder acquired non-determinism; do not relax the assertion.+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()+ await reconciler.reconcile(recipeWithWebhook())++ const sealed = new Map<string, unknown>()+ for (const call of mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls) {+ const body = (call[0] as { body: k8s.V1NetworkPolicy }).body+ sealed.set(body.metadata!.name!, JSON.parse(JSON.stringify(body)))+ }+ expect(sealed.size).toBe(3)++ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(+ ({ name }: { name: string }) => {+ const previous = sealed.get(name) as k8s.V1NetworkPolicy | undefined+ if (!previous) return Promise.resolve({ metadata: { name, resourceVersion: '1' } })+ return Promise.resolve({+ ...previous,+ metadata: { ...previous.metadata, resourceVersion: '9' },+ })+ }+ )+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()++ await reconciler.reconcile(recipeWithWebhook())++ for (const name of sealed.keys()) {+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(+ expect.objectContaining({ name })+ )+ }+ })+it('fail-closed when secretRef is missing: no gateway resources, condition set, phase degraded', async () => {mockAppsApi.createNamespacedDeployment.mockClear()mockCoreApi.createNamespacedService.mockClear()diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 790bee8f8..adcb5a37c 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -3031,28 +3031,19 @@ export class WorkflowRecipeReconciler {`WebhookGateway Service "${gatewayServiceName(recipeName)}"`)+ // Same spec-hash gate as every other WRC NetworkPolicy (issue #575): these+ // three are rebuilt verbatim from the recipe on every pass, so the inline+ // replace was three no-op PUTs per reconcile. The wr-intdep-* ownership+ // assertion inside applyNetworkPolicy returns immediately for these — their+ // policy-type label differs — so routing through it changes the write gate and+ // nothing else. The log label changes from `WebhookGateway NetworkPolicy "x"`+ // to `NetworkPolicy "x" in <ns>`, which no test, alert or doc consumes.for (const policy of [built.proxyIngressPolicy,built.handlerEgressPolicy,built.handlerIngressPolicy,]) {- const policyName = policy.metadata!.name!- await this.createOrReplace(- () => this.networkingApi.createNamespacedNetworkPolicy({ namespace: ns, body: policy }),- async () => {- const existing = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace: ns,- })- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion- return this.networkingApi.replaceNamespacedNetworkPolicy({- name: policyName,- namespace: ns,- body: policy,- })- },- `WebhookGateway NetworkPolicy "${policyName}"`- )+ await this.applyNetworkPolicy(policy, ns)}}@@ -3763,8 +3754,12 @@ export class WorkflowRecipeReconciler {/*** True when the existing object already carries the desired spec-hash, so the- * replace would be a no-op. A read failure returns false (fall through to- * replace) — never skip an update we cannot prove is unnecessary.+ * replace would be a no-op. Anything thrown by `readExisting` — a read failure,+ * or an invariant the caller asserts on what it just read — returns false and+ * falls through to the replace: never skip an update we cannot prove is+ * unnecessary. Callers whose `readExisting` asserts invariants must re-assert on+ * their own read in `replaceFn`: this catch is total by design (see+ * `applyNetworkPolicy`).*/private async applyIsNoop(idempotency: {manifest: { metadata?: { annotations?: { [key: string]: string } } }@@ -4876,8 +4871,37 @@ export class WorkflowRecipeReconciler {}}+ /**+ * Apply a WRC-authored NetworkPolicy through the spec-hash idempotency gate+ * (issue #575). Every family funnelling through here — ui-ingress, wl-ingress,+ * wr-intdep-*, oauth-broker egress, webhook-gateway, and wl-egress once #299 has+ * decided a write is due — is rebuilt deterministically from the recipe on every+ * pass, so the unconditional replace was one PUT per policy per ~16s that etcd+ * never recorded: pure apiserver CPU and priority-and-fairness budget.+ * Deployment, Service, CronJob, Job and DaemonSet already opted in; this was the+ * kind whose skip counter stayed at zero.+ *+ * The wr-intdep-* ownership assertion runs INSIDE `readExisting`, i.e. on the+ * very read the gate evaluates, so a spec-hash match can never shortcut it.+ * `applyIsNoop` swallows EVERY error `readExisting` throws by design — a read+ * failure must fall open to the replace — so the veto reaches the caller through+ * `replaceFn`, which re-reads and re-asserts. The read is cached only AFTER the+ * assertion passed: the replace path can never inherit a vetoed object, and the+ * PUT carries the resourceVersion of the object that passed, so the apiserver+ * rejects any change in between.+ */private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy, namespace: string): Promise<void> {const policyName = policy.metadata!.name!+ let existing: k8s.V1NetworkPolicy | undefined+ const readExistingAndAssertOwnership = async (): Promise<k8s.V1NetworkPolicy> => {+ const fetched = await this.networkingApi.readNamespacedNetworkPolicy({+ name: policyName,+ namespace,+ })+ this.assertInternalDependencyPolicyOwnership(policy, fetched, namespace)+ existing = fetched+ return fetched+ }await this.createOrReplace(() =>this.networkingApi.createNamespacedNetworkPolicy({@@ -4885,19 +4909,18 @@ export class WorkflowRecipeReconciler {body: policy,}),async () => {- const existing = await this.networkingApi.readNamespacedNetworkPolicy({- name: policyName,- namespace,- })- this.assertInternalDependencyPolicyOwnership(policy, existing, namespace)- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion+ // `existing` is unset when the gate's read failed OR its assertion vetoed+ // (applyIsNoop swallows both); re-read so the veto surfaces from here, loud.+ const current = existing ?? (await readExistingAndAssertOwnership())+ policy.metadata!.resourceVersion = current.metadata?.resourceVersionreturn this.networkingApi.replaceNamespacedNetworkPolicy({name: policyName,namespace,body: policy,})},- `NetworkPolicy "${policyName}" in ${namespace}`+ `NetworkPolicy "${policyName}" in ${namespace}`,+ { manifest: policy, readExisting: readExistingAndAssertOwnership })}