Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): apply NetworkPolicies through the spec-hash gate (#575)

Alfredo Lopez11db811887ee
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 910a6999d..97bd36ae3 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -302,7 +302,10 @@ kubectl delete workflowrecipe my-nginx -n sandbox-recipes
## Reconcile idempotency & drift behavior
WRC reconciles every recipe on a periodic resync. Each managed object (Deployment,
-StatefulSet, headless Service, CronJob, Job, DaemonSet, and the McpServer CRD) is
+StatefulSet, headless Service, CronJob, Job, DaemonSet, the McpServer CRD, and every
+WRC-authored NetworkPolicy applied through `applyNetworkPolicy` — `ui-ingress-*`,
+`wl-ingress-*`, `wr-intdep-*`, `wf-*-oauth-broker-egress`, the three webhook-gateway
+policies, and `wl-egress-*` once issue #299 has decided a write is due) is
applied through a **spec-hash idempotency gate**: WRC stamps a `clerum.io/spec-hash`
annotation (a hash of the _desired_ manifest) and, on the already-exists path, **skips
the write entirely when the existing object's stamped hash matches**. This keeps
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index bd7095d74..fbee3b101 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.339",
+ "version": "0.1.340",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.339",
+ "version": "0.1.340",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index c043e5cb5..7747bce78 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.339",
+ "version": "0.1.340",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 1018b40a9..cfccea8ec 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -18,6 +18,7 @@ import {
resolveScopedWorkloadResourceName,
resolveWorkloadRuntimeResourceName,
} from './resourceBuilder'
+import { stampSpecHash } from './specHash'
import {
TRANSIENT_REQUEUE_BASE_MS,
WORKFLOW_PROGRESS_REQUEUE_BASE_MS,
@@ -769,6 +770,212 @@ describe('WorkflowRecipeReconciler', () => {
])
})
+ // ─── issue #575: the spec-hash gate applyNetworkPolicy never opted into ──────
+ //
+ // `createOrReplace` takes a fourth `idempotency` argument that stamps the
+ // desired manifest with a SHA-256 of its content and compares it against the
+ // annotation on the live object before replacing. Five callers pass it —
+ // Deployment, headless Service, CronJob, Job, DaemonSet. `applyNetworkPolicy`
+ // did not, so every WRC NetworkPolicy funnelling through it was rewritten on
+ // every reconcile pass without comparing anything: ~12 200 PUTs/hour of objects
+ // the apiserver then discarded as unchanged.
+ //
+ // The ownership assertion is why this is not a one-line change. It runs inside
+ // the replace path today; a gate that returns first would skip it, so it moves
+ // into `readExisting` — the very read the gate evaluates — and the result is
+ // cached only AFTER it passes, so the replace path can never inherit a vetoed
+ // object.
+ describe('applyNetworkPolicy spec-hash gate (issue #575)', () => {
+ const SPEC_HASH = 'clerum.io/spec-hash'
+ const NS = 'sandbox-recipes'
+ const clone = <T>(v: T): T => JSON.parse(JSON.stringify(v)) as T
+ type Priv = { applyNetworkPolicy: (p: k8s.V1NetworkPolicy, ns: string) => Promise<void> }
+ const apply = (p: k8s.V1NetworkPolicy) =>
+ (reconciler as unknown as Priv).applyNetworkPolicy(p, NS)
+
+ const plainPolicy = (): k8s.V1NetworkPolicy => ({
+ apiVersion: 'networking.k8s.io/v1',
+ kind: 'NetworkPolicy',
+ metadata: {
+ name: 'wl-ingress-r-w',
+ namespace: NS,
+ labels: { 'clerum.io/managed-by': 'workflow-recipes', 'clerum.io/recipe': 'r' },
+ },
+ spec: {
+ podSelector: { matchLabels: { 'clerum.io/workload': 'w' } },
+ policyTypes: ['Ingress'],
+ ingress: [{ ports: [{ port: 8080, protocol: 'TCP' }] }],
+ },
+ })
+
+ const intdepPolicy = (): k8s.V1NetworkPolicy => ({
+ ...plainPolicy(),
+ metadata: {
+ name: 'wr-intdep-egress-r-w',
+ namespace: NS,
+ labels: {
+ 'clerum.io/managed-by': 'workflow-recipes',
+ 'clerum.io/recipe': 'r',
+ 'clerum.io/policy-type': 'internal-dependency',
+ },
+ },
+ })
+
+ const ownedExisting = (extra: Record<string, unknown> = {}) => ({
+ metadata: {
+ name: 'wl-ingress-r-w',
+ resourceVersion: '9',
+ labels: plainPolicy().metadata!.labels,
+ ...extra,
+ },
+ })
+
+ it('T1 does NOT replace a NetworkPolicy whose spec-hash is unchanged, and says so', async () => {
+ await apply(plainPolicy())
+ const sealed = clone(
+ (
+ mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls.at(-1)![0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ )
+ expect(sealed.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()
+
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ ...sealed,
+ metadata: { ...sealed.metadata, resourceVersion: '9' },
+ })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockClear()
+ const logSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined)
+ try {
+ await apply(plainPolicy())
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ // The skip line is load-bearing, not decoration: it is what keeps the
+ // ~16s reconcile cadence visible once the writes stop. A silent gate
+ // would make the symptom of #460 invisible.
+ expect(logSpy.mock.calls.flat().join(' ')).toContain(
+ 'NetworkPolicy "wl-ingress-r-w" in sandbox-recipes unchanged (spec-hash match); skipping update'
+ )
+ } finally {
+ logSpy.mockRestore()
+ }
+ })
+
+ it('T2 replaces when the spec-hash differs, reading once and using the live resourceVersion', async () => {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(
+ ownedExisting({ annotations: { [SPEC_HASH]: 'stale' } })
+ )
+
+ await apply(plainPolicy())
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ expect(body.metadata?.resourceVersion).toBe('9')
+ expect(body.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()
+ // One read serves both the gate and the replace, so the replace cannot act
+ // on a different resourceVersion than the one the gate evaluated.
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ })
+
+ it('T3 writes once to seal a pre-existing policy with no spec-hash, then skips', async () => {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(ownedExisting())
+
+ await apply(plainPolicy())
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const sealed = clone(
+ (
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0] as {
+ body: k8s.V1NetworkPolicy
+ }
+ ).body
+ )
+ expect(sealed.metadata?.annotations?.[SPEC_HASH]).toBeTruthy()
+
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(sealed)
+
+ await apply(plainPolicy())
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ })
+
+ it('T4 does not let a matching spec-hash shortcut the ownership assertion', async () => {
+ const desired = intdepPolicy()
+ const sameHash = stampSpecHash(clone(desired))
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ // Foreign owner, carrying a hash the gate would happily match on.
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue({
+ metadata: {
+ name: 'wr-intdep-egress-r-w',
+ resourceVersion: '9',
+ annotations: { [SPEC_HASH]: sameHash },
+ labels: { 'clerum.io/managed-by': 'hcc' },
+ },
+ })
+
+ await expect(apply(desired)).rejects.toThrow(
+ /Refusing to replace NetworkPolicy "wr-intdep-egress-r-w"/
+ )
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
+ // Two reads: the gate's read raises the veto, `applyIsNoop` treats any
+ // throw as "cannot prove this is a no-op" and falls through, and the replace
+ // path re-reads and re-asserts, where the veto propagates. The write never
+ // happens either way. What this test rules out is the assertion living ONLY
+ // in the replace path: there, a matching hash would make the gate skip and
+ // this call would resolve instead of rejecting.
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+
+ it('T5 falls through to the replace when the gate read fails with an HTTP status', async () => {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 500 })
+ .mockResolvedValueOnce(ownedExisting())
+
+ await apply(plainPolicy())
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+
+ it('T5c falls through on a codeless read error, the shape `:1354` pins for Deployment', async () => {
+ // The rule cannot key on the shape of the error. A bare Error with no code
+ // is a legitimate read failure — the existing Deployment test injects
+ // exactly `new Error('transient read timeout')` and asserts the write still
+ // happens — so `applyIsNoop` swallows everything and falls open. A version
+ // of this gate that classified errors instead broke that test; this pins the
+ // behaviour for NetworkPolicy so the same idea cannot come back quietly.
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce(new Error('transient read timeout'))
+ .mockResolvedValueOnce(ownedExisting())
+
+ await apply(plainPolicy())
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+
+ it('T5b falls through to the replace on a retryable transport error', async () => {
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy
+ .mockRejectedValueOnce({ code: 'ECONNRESET' })
+ .mockResolvedValueOnce(ownedExisting())
+
+ await apply(plainPolicy())
+
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
+ })
+ })
+
describe('idempotent apply (stops generation churn)', () => {
const SPEC_HASH = 'clerum.io/spec-hash'
const clone = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T
@@ -9191,6 +9398,47 @@ describe('WorkflowRecipeReconciler', () => {
expect(allFalse).toBe(true)
})
+ it('T6 (#575) applies the 3 gateway NetworkPolicies through the spec-hash gate', async () => {
+ // Two jobs. First: the gateway policies used to be written by an inline
+ // create-or-replace with no gate, so they were three no-op PUTs per pass.
+ // Second, and the reason this test is here rather than asserted by
+ // inspection: it proves the gateway builder is DETERMINISTIC. A gate only
+ // closes if the manifest hashes the same on the next pass, and a builder
+ // that iterated a Set or stamped a timestamp would leave the gate open
+ // forever — the failure mode of #413 and #579. If this test goes red, the
+ // builder acquired non-determinism; do not relax the assertion.
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockClear()
+ await reconciler.reconcile(recipeWithWebhook())
+
+ const sealed = new Map<string, unknown>()
+ for (const call of mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls) {
+ const body = (call[0] as { body: k8s.V1NetworkPolicy }).body
+ sealed.set(body.metadata!.name!, JSON.parse(JSON.stringify(body)))
+ }
+ expect(sealed.size).toBe(3)
+
+ mockNetworkingApi.createNamespacedNetworkPolicy.mockRejectedValue({ code: 409 })
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(
+ ({ name }: { name: string }) => {
+ const previous = sealed.get(name) as k8s.V1NetworkPolicy | undefined
+ if (!previous) return Promise.resolve({ metadata: { name, resourceVersion: '1' } })
+ return Promise.resolve({
+ ...previous,
+ metadata: { ...previous.metadata, resourceVersion: '9' },
+ })
+ }
+ )
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+
+ await reconciler.reconcile(recipeWithWebhook())
+
+ for (const name of sealed.keys()) {
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalledWith(
+ expect.objectContaining({ name })
+ )
+ }
+ })
+
it('fail-closed when secretRef is missing: no gateway resources, condition set, phase degraded', async () => {
mockAppsApi.createNamespacedDeployment.mockClear()
mockCoreApi.createNamespacedService.mockClear()
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 790bee8f8..adcb5a37c 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -3031,28 +3031,19 @@ export class WorkflowRecipeReconciler {
`WebhookGateway Service "${gatewayServiceName(recipeName)}"`
)
+ // Same spec-hash gate as every other WRC NetworkPolicy (issue #575): these
+ // three are rebuilt verbatim from the recipe on every pass, so the inline
+ // replace was three no-op PUTs per reconcile. The wr-intdep-* ownership
+ // assertion inside applyNetworkPolicy returns immediately for these — their
+ // policy-type label differs — so routing through it changes the write gate and
+ // nothing else. The log label changes from `WebhookGateway NetworkPolicy "x"`
+ // to `NetworkPolicy "x" in <ns>`, which no test, alert or doc consumes.
for (const policy of [
built.proxyIngressPolicy,
built.handlerEgressPolicy,
built.handlerIngressPolicy,
]) {
- const policyName = policy.metadata!.name!
- await this.createOrReplace(
- () => this.networkingApi.createNamespacedNetworkPolicy({ namespace: ns, body: policy }),
- async () => {
- const existing = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace: ns,
- })
- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
- return this.networkingApi.replaceNamespacedNetworkPolicy({
- name: policyName,
- namespace: ns,
- body: policy,
- })
- },
- `WebhookGateway NetworkPolicy "${policyName}"`
- )
+ await this.applyNetworkPolicy(policy, ns)
}
}
@@ -3763,8 +3754,12 @@ export class WorkflowRecipeReconciler {
/**
* True when the existing object already carries the desired spec-hash, so the
- * replace would be a no-op. A read failure returns false (fall through to
- * replace) — never skip an update we cannot prove is unnecessary.
+ * replace would be a no-op. Anything thrown by `readExisting` — a read failure,
+ * or an invariant the caller asserts on what it just read — returns false and
+ * falls through to the replace: never skip an update we cannot prove is
+ * unnecessary. Callers whose `readExisting` asserts invariants must re-assert on
+ * their own read in `replaceFn`: this catch is total by design (see
+ * `applyNetworkPolicy`).
*/
private async applyIsNoop(idempotency: {
manifest: { metadata?: { annotations?: { [key: string]: string } } }
@@ -4876,8 +4871,37 @@ export class WorkflowRecipeReconciler {
}
}
+ /**
+ * Apply a WRC-authored NetworkPolicy through the spec-hash idempotency gate
+ * (issue #575). Every family funnelling through here — ui-ingress, wl-ingress,
+ * wr-intdep-*, oauth-broker egress, webhook-gateway, and wl-egress once #299 has
+ * decided a write is due — is rebuilt deterministically from the recipe on every
+ * pass, so the unconditional replace was one PUT per policy per ~16s that etcd
+ * never recorded: pure apiserver CPU and priority-and-fairness budget.
+ * Deployment, Service, CronJob, Job and DaemonSet already opted in; this was the
+ * kind whose skip counter stayed at zero.
+ *
+ * The wr-intdep-* ownership assertion runs INSIDE `readExisting`, i.e. on the
+ * very read the gate evaluates, so a spec-hash match can never shortcut it.
+ * `applyIsNoop` swallows EVERY error `readExisting` throws by design — a read
+ * failure must fall open to the replace — so the veto reaches the caller through
+ * `replaceFn`, which re-reads and re-asserts. The read is cached only AFTER the
+ * assertion passed: the replace path can never inherit a vetoed object, and the
+ * PUT carries the resourceVersion of the object that passed, so the apiserver
+ * rejects any change in between.
+ */
private async applyNetworkPolicy(policy: k8s.V1NetworkPolicy, namespace: string): Promise<void> {
const policyName = policy.metadata!.name!
+ let existing: k8s.V1NetworkPolicy | undefined
+ const readExistingAndAssertOwnership = async (): Promise<k8s.V1NetworkPolicy> => {
+ const fetched = await this.networkingApi.readNamespacedNetworkPolicy({
+ name: policyName,
+ namespace,
+ })
+ this.assertInternalDependencyPolicyOwnership(policy, fetched, namespace)
+ existing = fetched
+ return fetched
+ }
await this.createOrReplace(
() =>
this.networkingApi.createNamespacedNetworkPolicy({
@@ -4885,19 +4909,18 @@ export class WorkflowRecipeReconciler {
body: policy,
}),
async () => {
- const existing = await this.networkingApi.readNamespacedNetworkPolicy({
- name: policyName,
- namespace,
- })
- this.assertInternalDependencyPolicyOwnership(policy, existing, namespace)
- policy.metadata!.resourceVersion = existing.metadata?.resourceVersion
+ // `existing` is unset when the gate's read failed OR its assertion vetoed
+ // (applyIsNoop swallows both); re-read so the veto surfaces from here, loud.
+ const current = existing ?? (await readExistingAndAssertOwnership())
+ policy.metadata!.resourceVersion = current.metadata?.resourceVersion
return this.networkingApi.replaceNamespacedNetworkPolicy({
name: policyName,
namespace,
body: policy,
})
},
- `NetworkPolicy "${policyName}" in ${namespace}`
+ `NetworkPolicy "${policyName}" in ${namespace}`,
+ { manifest: policy, readExisting: readExistingAndAssertOwnership }
)
}