Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): harden ui policy metadata and review witnesses

Alfredo Lopez650f74e03125
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 5fe6d798d..0cd5e2559 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -315,7 +315,10 @@ it cannot hide later drift.
- `ui-egress-*` and the `coordinator-to-gfs` policy keep their existing dedicated
comparisons (#299 and #579 respectively); they do not use the recipe NetworkPolicy
- helper.
+ live-convergence helper. The dedicated UI egress writer still preserves external
+ metadata and rejects foreign lifecycle owners or terminating objects before a
+ replacement, using the freshly read resourceVersion. This does not convert its
+ create-first write path into the read-first no-write contract above.
- `wl-egress-*` still uses the #299 content/renewal prefilter before the live convergence
helper. External/mixed egress carries temporal resolution state; a cluster-local-only
policy does not. In both cases, once the prefilter requires a write, the downstream
@@ -325,6 +328,9 @@ it cannot hide later drift.
**Operational consequences:**
+- The zero-write E2E observer covers only the six live-convergence families listed
+ above. It does not certify UI egress or coordinator-to-GFS: their dedicated
+ comparisons and generic resource logs are not that observer's event contract.
- **Logs:** the live-convergence helper and the `wl-egress-*` prefilter use the structured
WRC logger. In steady state expect `network policy unchanged; skipping update` or
`network policy egress set unchanged; skipping live apply` with `policy`, `namespace`
diff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
index fee52f1c1..d451e23d5 100755
--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
@@ -32,6 +32,7 @@ DB_DEPLOYMENT=""
OTHER_DB_DEPLOYMENT=""
UI_INGRESS_POLICY="ui-ingress-${RECIPE_NAME}-${API_ID}"
+UI_EGRESS_POLICY="ui-egress-${RECIPE_NAME}"
WL_EGRESS_POLICY="wl-egress-${RECIPE_NAME}-${API_ID}"
WL_INGRESS_POLICY="wl-ingress-${RECIPE_NAME}-${DB_ID}"
@@ -307,6 +308,19 @@ wrc_assert_http_allowed "sandbox UI route recovered after NetworkPolicy repair"
wrc_assert_http_allowed "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
header "Phase 5 — contraction preserves external metadata and live routes"
+# The dedicated UI egress writer also sees the shared metadata prefilter.
+# Remove one authored identity label without changing any network permission;
+# its repair must preserve unrelated metadata rather than replacing it blindly.
+ui_egress_live="$(kctl get networkpolicy "$UI_EGRESS_POLICY" -n "$SANDBOX_UI_NS" -o json)"
+printf '%s' "$ui_egress_live" | wrc_record_owned
+ui_egress_uid="$(printf '%s' "$ui_egress_live" | jq -er '.metadata.uid')"
+ui_egress_patch="$(printf '%s' "$ui_egress_live" | jq -ce --arg run "$E2E_RUN_ID" '
+ [{op:"test",path:"/metadata/uid",value:.metadata.uid},
+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},
+ {op:"remove",path:"/metadata/labels/clerum.io~1recipe-namespace"},
+ {op:"add",path:"/metadata/annotations",value:((.metadata.annotations // {}) + {"e2e.invalid/external-annotation":$run})},
+ {op:"add",path:"/metadata/labels/e2e.invalid~1external-label",value:$run}]')"
+kctl patch networkpolicy "$UI_EGRESS_POLICY" -n "$SANDBOX_UI_NS" --type=json -p "$ui_egress_patch" >/dev/null
# Add only an unused port to each fixture ingress. Repair must remove that
# permission without discarding another controller's metadata. This exercises
# the pre-DNS contraction writer, not merely the final additive apply.
@@ -332,6 +346,16 @@ for policy in "$UI_INGRESS_POLICY" "$WL_INGRESS_POLICY"; do
if [ "$policy" = "$WL_INGRESS_POLICY" ]; then HELD_POLICY_FINALIZER=1; fi
done
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
+kctl get networkpolicy "$UI_EGRESS_POLICY" -n "$SANDBOX_UI_NS" -o json | jq -e \
+ --arg run "$E2E_RUN_ID" --arg uid "$ui_egress_uid" --arg recipe_ns "$WORKFLOW_RECIPE_NS" '
+ .metadata.uid == $uid and
+ .metadata.labels["clerum.io/recipe-namespace"] == $recipe_ns and
+ .metadata.labels["e2e.invalid/external-label"] == $run and
+ .metadata.annotations["e2e.invalid/external-annotation"] == $run' >/dev/null || {
+ fail 'Dedicated UI egress repair failed to restore authored metadata while preserving external metadata'
+ exit 1
+}
+ok 'Dedicated UI egress repair preserved external metadata and policy identity'
wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$UI_INGRESS_POLICY" "$ui_hash" 120
wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$wl_ingress_hash" 120
for policy in "$UI_INGRESS_POLICY" "$WL_INGRESS_POLICY"; do
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 85a36e10b..ea302c968 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 13e1eff8c..58d13b083 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.348",
+ "version": "0.1.349",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
index 75f7a09ad..ee6b4079a 100644
--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
@@ -200,6 +200,9 @@ export function networkPolicyMatchesDesired(
)
)
} catch {
+ // An uncomparable live object cannot prove convergence. Keep it on the
+ // guarded repair path rather than treating a failed comparison as a no-op;
+ // ownership/lifecycle checks and resourceVersion fencing still apply.
return false
}
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 2521fc95f..f51dfa40e 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -509,6 +509,32 @@ describe('WorkflowRecipeReconciler', () => {
})
})
+ function captureNetworkPolicyLogs(): {
+ entries: Array<Record<string, unknown>>
+ restore: () => void
+ } {
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const entries: Array<Record<string, unknown>> = []
+ const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
+ for (const line of text.split('\n')) {
+ if (!line.startsWith('{')) continue
+ const parsed = JSON.parse(line) as Record<string, unknown>
+ if (String(parsed.msg).startsWith('network policy')) entries.push(parsed)
+ }
+ return true
+ }) as unknown as typeof process.stdout.write)
+ return {
+ entries,
+ restore: () => {
+ spy.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ },
+ }
+ }
+
describe('#567 regression: policy contraction', () => {
describe('#580 contraction lifecycle regression', () => {
const namespace = 'sandbox-recipes'
@@ -567,11 +593,25 @@ describe('WorkflowRecipeReconciler', () => {
namespace,
recipeName: 'test-recipe',
})
- await pipeline.applyNetworkPolicy(desired, namespace, {
- family,
- recipeName: 'test-recipe',
- existing: contracted,
- })
+ const captured = captureNetworkPolicyLogs()
+ try {
+ await pipeline.applyNetworkPolicy(desired, namespace, {
+ family,
+ recipeName: 'test-recipe',
+ existing: contracted,
+ })
+ expect(captured.entries).toContainEqual(
+ expect.objectContaining({
+ level: 'info',
+ msg: 'network policy unchanged; skipping update',
+ policy: name,
+ namespace,
+ family,
+ })
+ )
+ } finally {
+ captured.restore()
+ }
expect(live.get(name)?.metadata).toMatchObject({
labels: { 'external.example/team': 'payments' },
annotations: { 'external.example/review': 'retained' },
@@ -1857,32 +1897,6 @@ describe('WorkflowRecipeReconciler', () => {
expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
})
- function captureNetworkPolicyLogs(): {
- entries: Array<Record<string, unknown>>
- restore: () => void
- } {
- const previousLevel = process.env.LOG_LEVEL
- process.env.LOG_LEVEL = 'info'
- const entries: Array<Record<string, unknown>> = []
- const spy = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
- const text = typeof chunk === 'string' ? chunk : Buffer.from(chunk as Uint8Array).toString()
- for (const line of text.split('\n')) {
- if (!line.startsWith('{')) continue
- const parsed = JSON.parse(line) as Record<string, unknown>
- if (String(parsed.msg).startsWith('network policy')) entries.push(parsed)
- }
- return true
- }) as unknown as typeof process.stdout.write)
- return {
- entries,
- restore: () => {
- spy.mockRestore()
- if (previousLevel === undefined) delete process.env.LOG_LEVEL
- else process.env.LOG_LEVEL = previousLevel
- },
- }
- }
-
it('T1 skips a live-equivalent policy and emits a structured liveness witness', async () => {
const desired = plainPolicy()
const live = livePolicy(desired, { annotations: { [SPEC_HASH]: 'legacy-seal' } })
@@ -2004,6 +2018,7 @@ describe('WorkflowRecipeReconciler', () => {
await apply(desired)
expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
const created = clone(
(
mockNetworkingApi.createNamespacedNetworkPolicy.mock.calls[0][0] as {
@@ -2017,6 +2032,7 @@ describe('WorkflowRecipeReconciler', () => {
mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(livePolicy(created))
await apply(desired)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(2)
expect(mockNetworkingApi.createNamespacedNetworkPolicy).not.toHaveBeenCalled()
expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
diff --git a/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts b/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts
new file mode 100644
index 000000000..0ed23dae3
--- /dev/null
+++ b/workflow-recipes/src/reconciler/uiEgressMetadata.test.ts
@@ -0,0 +1,174 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import * as k8s from '@kubernetes/client-node'
+import { loadConfig } from '../config'
+import type { WorkflowRecipeCRD } from '../types'
+import { captureLogger } from './__tests__/captureLogger'
+import { buildUiEgressNetworkPolicy } from './resourceBuilder'
+import { WorkflowRecipeReconciler } from './workflowRecipeReconciler'
+
+afterEach(() => vi.restoreAllMocks())
+
+function uiPolicyFixture() {
+ const config = loadConfig()
+ const recipe: WorkflowRecipeCRD = {
+ apiVersion: 'clerum.io/v1alpha1',
+ kind: 'WorkflowRecipe',
+ metadata: { name: 'ui-metadata', namespace: config.sandboxNamespace, uid: 'recipe-uid' },
+ spec: {
+ workloads: [{ id: 'backend', type: 'deployment', image: 'nginx:1.30.1-alpine', port: 8080 }],
+ ui: {
+ workloadRef: 'backend',
+ port: 8080,
+ egress: { internal: [{ workloadRef: 'backend', port: 8080 }] },
+ },
+ },
+ }
+ let live = buildUiEgressNetworkPolicy(
+ recipe,
+ config.sandboxUiNamespace,
+ config.sandboxNamespace,
+ []
+ )!
+ live.metadata = {
+ ...live.metadata,
+ uid: 'policy-uid',
+ resourceVersion: '7',
+ labels: { ...live.metadata!.labels, 'external.example/owner': 'retain' },
+ annotations: { 'external.example/note': 'retain' },
+ finalizers: ['external.example/hold'],
+ }
+ const read = vi.fn(async () => structuredClone(live))
+ const create = vi.fn(async (): Promise<unknown> => {
+ throw { code: 409 }
+ })
+ const replace = vi.fn(async ({ body }: { body: k8s.V1NetworkPolicy }) => {
+ expect(body.metadata?.resourceVersion).toBe(live.metadata?.resourceVersion)
+ live = structuredClone(body)
+ return structuredClone(live)
+ })
+ const networking = {
+ readNamespacedNetworkPolicy: read,
+ createNamespacedNetworkPolicy: create,
+ replaceNamespacedNetworkPolicy: replace,
+ deleteNamespacedNetworkPolicy: vi.fn(),
+ }
+ const kc = new k8s.KubeConfig()
+ vi.spyOn(kc, 'makeApiClient').mockImplementation((() => networking) as typeof kc.makeApiClient)
+ const reconciler = new WorkflowRecipeReconciler(kc, config, {
+ fqdnLookup: async () => {
+ throw new Error('Internal-only UI must not resolve DNS')
+ },
+ })
+ const reconcile = () =>
+ (
+ reconciler as unknown as {
+ reconcileUiEgressPolicy(recipe: WorkflowRecipeCRD): Promise<void>
+ }
+ ).reconcileUiEgressPolicy(recipe)
+ return { recipe, read, create, replace, reconcile, live: () => live }
+}
+
+describe('UI egress dedicated writer metadata and lifecycle', () => {
+ it('keeps an equivalent UI policy read-only with a positive no-op witness', async () => {
+ const fixture = uiPolicyFixture()
+ const logs = captureLogger('info')
+
+ await fixture.reconcile()
+
+ expect(fixture.read).toHaveBeenCalledTimes(1)
+ expect(logs).toHaveBeenCalledWith('NetworkPolicy egress set unchanged; no-op', {
+ policyName: fixture.live().metadata!.name,
+ ns: fixture.live().metadata!.namespace,
+ })
+ expect(fixture.create).not.toHaveBeenCalled()
+ expect(fixture.replace).not.toHaveBeenCalled()
+ })
+
+ it('creates a missing UI policy through the dedicated writer', async () => {
+ const fixture = uiPolicyFixture()
+ fixture.read.mockRejectedValueOnce({ code: 404 })
+ fixture.create.mockResolvedValueOnce({})
+
+ await fixture.reconcile()
+
+ expect(fixture.read).toHaveBeenCalledTimes(1)
+ expect(fixture.create).toHaveBeenCalledWith({
+ namespace: fixture.live().metadata!.namespace,
+ body: expect.objectContaining({ spec: fixture.live().spec }),
+ })
+ expect(fixture.replace).not.toHaveBeenCalled()
+ })
+
+ it.each(['clerum.io/recipe-namespace', 'clerum.io/recipe-name'])(
+ 'preserves external metadata while repairing missing %s',
+ async label => {
+ const fixture = uiPolicyFixture()
+ delete fixture.live().metadata!.labels![label]
+
+ await fixture.reconcile()
+
+ expect(fixture.read).toHaveBeenCalledTimes(2)
+ expect(fixture.create).toHaveBeenCalledTimes(1)
+ expect(fixture.replace).toHaveBeenCalledTimes(1)
+ expect(fixture.live().metadata).toMatchObject({
+ labels: {
+ 'external.example/owner': 'retain',
+ [label]: label.endsWith('namespace')
+ ? fixture.recipe.metadata.namespace
+ : fixture.recipe.metadata.name,
+ },
+ annotations: { 'external.example/note': 'retain' },
+ finalizers: ['external.example/hold'],
+ })
+ }
+ )
+
+ it('preserves external metadata when current intent adds a UI destination port', async () => {
+ const fixture = uiPolicyFixture()
+ fixture.recipe.spec.ui!.egress!.internal!.push({ workloadRef: 'backend', port: 9090 })
+
+ await fixture.reconcile()
+
+ expect(fixture.replace).toHaveBeenCalledTimes(1)
+ expect(
+ fixture.live().spec!.egress!.flatMap(rule => rule.ports!.map(port => port.port))
+ ).toEqual([8080, 9090])
+ expect(fixture.live().metadata).toMatchObject({
+ labels: { 'external.example/owner': 'retain' },
+ annotations: { 'external.example/note': 'retain' },
+ finalizers: ['external.example/hold'],
+ })
+ })
+
+ it.each(['terminating', 'controller-owner', 'non-controller-owner'])(
+ 'rejects %s appearing between contraction and final replacement',
+ async lifecycle => {
+ const fixture = uiPolicyFixture()
+ delete fixture.live().metadata!.labels!['clerum.io/recipe-namespace']
+ fixture.read
+ .mockImplementationOnce(async () => structuredClone(fixture.live()))
+ .mockImplementationOnce(async () => {
+ const raced = structuredClone(fixture.live())
+ if (lifecycle === 'terminating') raced.metadata!.deletionTimestamp = new Date()
+ else
+ raced.metadata!.ownerReferences = [
+ {
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ name: 'foreign',
+ uid: 'foreign-uid',
+ controller: lifecycle === 'controller-owner',
+ },
+ ]
+ return raced
+ })
+
+ await expect(fixture.reconcile()).rejects.toThrow(
+ lifecycle === 'terminating' ? 'terminating' : 'owner-reference-mismatch'
+ )
+ expect(fixture.read).toHaveBeenCalledTimes(2)
+ expect(fixture.create).toHaveBeenCalledTimes(1)
+ expect(fixture.replace).not.toHaveBeenCalled()
+ }
+ )
+})
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 27c6d9100..0cc3f51f9 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -5279,6 +5279,28 @@ export class WorkflowRecipeReconciler {
}
}
+ private assertMutableRecipeNetworkPolicy(
+ snapshot: k8s.V1NetworkPolicy,
+ recipeName: string,
+ policyName: string,
+ namespace: string
+ ): void {
+ // Recipe-scoped policies are label-owned, not garbage-collection owned.
+ // Check every freshly read snapshot, including final apply after DNS.
+ if (snapshot.metadata?.deletionTimestamp) {
+ throw new RetryableReconcileError(
+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`
+ )
+ }
+ this.assertRecipeNetworkPolicyOwnership(snapshot, recipeName, policyName, namespace)
+ const ownership = classifyOwnerlessNetworkPolicyOwnership(snapshot)
+ if (ownership.kind === 'conflict') {
+ throw new NetworkPolicyOwnershipConflictError(
+ `Refusing to mutate NetworkPolicy "${policyName}" in ${namespace}: ${ownership.reason}`
+ )
+ }
+ }
+
/**
* Apply a strictly subtractive pre-DNS transition against the exact snapshot
* that was read. This path never creates a policy and therefore cannot add an
@@ -5297,24 +5319,7 @@ export class WorkflowRecipeReconciler {
if (!policyName || !resourceVersion) {
throw new Error('NetworkPolicy contraction requires policy name and resourceVersion')
}
- const assertMutableSnapshot = (snapshot: k8s.V1NetworkPolicy): void => {
- // These recipe-scoped policies are label-owned, not garbage-collection
- // owned. Apply the same lifecycle veto as final convergence before even
- // a no-op/delete, and again before trusting or revoking the read-back.
- if (snapshot.metadata?.deletionTimestamp) {
- throw new RetryableReconcileError(
- `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`
- )
- }
- this.assertRecipeNetworkPolicyOwnership(snapshot, recipeName, policyName, namespace)
- const ownership = classifyOwnerlessNetworkPolicyOwnership(snapshot)
- if (ownership.kind === 'conflict') {
- throw new NetworkPolicyOwnershipConflictError(
- `Refusing to mutate NetworkPolicy "${policyName}" in ${namespace}: ${ownership.reason}`
- )
- }
- }
- assertMutableSnapshot(existing)
+ this.assertMutableRecipeNetworkPolicy(existing, recipeName, policyName, namespace)
if (desired === null) {
const uid = existing.metadata?.uid
@@ -5359,7 +5364,7 @@ export class WorkflowRecipeReconciler {
if (getErrorCode(error) === 404) return null
throw error
}
- assertMutableSnapshot(live)
+ this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)
const liveUid = live.metadata?.uid
const liveResourceVersion = live.metadata?.resourceVersion
if (
@@ -5429,21 +5434,18 @@ export class WorkflowRecipeReconciler {
}
throw error
}
- this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)
+ this.assertMutableRecipeNetworkPolicy(live, recipeName, policyName, namespace)
if (!live.metadata?.resourceVersion) {
throw new NetworkPolicyOwnershipConflictError(
`Refusing to replace NetworkPolicy "${policyName}" in ${namespace}: live resourceVersion is missing`
)
}
- policy.metadata = {
- ...(policy.metadata ?? {}),
- resourceVersion: live.metadata?.resourceVersion,
- }
+ const replacement = buildNetworkPolicyReplacement(policy, live)
try {
return await this.networkingApi.replaceNamespacedNetworkPolicy({
name: policyName,
namespace,
- body: policy,
+ body: replacement,
})
} catch (error: unknown) {
// This PUT is fenced by the freshly checked owner/resourceVersion.