Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): preserve policy ownership during contraction
Alfredo Lopezdae205f19be9
diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.shindex dff36bd82..1604e00eb 100755--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh@@ -243,10 +243,14 @@ wrc_assert_np_observation_clean() {return 1}events="$(printf '%s' "$logs" | wrc_log_events "$WRC_NP_OBSERVATION_SINCE")" || return 1- # Match writes by namespace/name even if a broken writer mislabels family.+ # Match every final-apply and pre-DNS contraction event by namespace/name:+ # contraction logs omit family, and a broken writer can mislabel it. Reject+ # the contraction message regardless of action so a new action fails closed.if printf '%s' "$events" | jq -e --argjson tracked "$WRC_NP_TRACKED" 'any(.[]; . as $event | any($tracked[]; .namespace == $event.namespace and .policy == $event.policy)- and ((.msg // "") | test("^network policy (created|replaced|create failed|replace failed|read failed)$")))' >/dev/null; then+ and (.msg == "contracted recipe network policy"+ or .msg == "network policy read unavailable; retrying once"+ or ((.msg // "") | test("^network policy (created|replaced|create failed|replace failed|read failed)$"))))' >/dev/null; thenfail 'NetworkPolicy window contains a write or failed operation, even if a no-op also occurred'return 1fidiff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.shindex 99d8afdc5..fee52f1c1 100755--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh@@ -306,7 +306,55 @@ ok "WRC repaired live spec drift for all three families"wrc_assert_http_allowed "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"wrc_assert_http_allowed "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"-header "Phase 5 — terminating race self-heals without another parent event"+header "Phase 5 — contraction preserves external metadata and live routes"+# Add only an unused port to each fixture ingress. Repair must remove that+# permission without discarding another controller's metadata. This exercises+# the pre-DNS contraction writer, not merely the final additive apply.+for policy in "$UI_INGRESS_POLICY" "$WL_INGRESS_POLICY"; do+ live="$(kctl get networkpolicy "$policy" -n "$SANDBOX_NS" -o json)"+ printf '%s' "$live" | wrc_record_owned+ if [ "$policy" = "$WL_INGRESS_POLICY" ]; then+ HELD_POLICY_UID="$(printf '%s' "$live" | jq -er '.metadata.uid')"+ else+ UI_CONTRACTION_UID="$(printf '%s' "$live" | jq -er '.metadata.uid')"+ fi+ contraction_patch="$(printf '%s' "$live" | jq -ce --arg run "$E2E_RUN_ID" \+ --arg policy "$policy" --arg held "$WL_INGRESS_POLICY" '+ [{op:"test",path:"/metadata/uid",value:.metadata.uid},+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},+ {op:"add",path:"/metadata/labels",value:((.metadata.labels // {}) + {"e2e.invalid/external-label":$run})},+ {op:"add",path:"/metadata/annotations",value:((.metadata.annotations // {}) + {"e2e.invalid/external-annotation":$run})},+ {op:"add",path:"/spec/ingress/0/ports/-",value:{protocol:"TCP",port:18443}}]+ + (if $policy == $held then+ [{op:"add",path:"/metadata/finalizers",value:((.metadata.finalizers // []) + ["e2e.invalid/hold-deletion"])}]+ else [] end)')"+ kctl patch networkpolicy "$policy" -n "$SANDBOX_NS" --type=json -p "$contraction_patch" >/dev/null+ if [ "$policy" = "$WL_INGRESS_POLICY" ]; then HELD_POLICY_FINALIZER=1; fi+done+wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120+wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$UI_INGRESS_POLICY" "$ui_hash" 120+wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$wl_ingress_hash" 120+for policy in "$UI_INGRESS_POLICY" "$WL_INGRESS_POLICY"; do+ expected_uid=$UI_CONTRACTION_UID+ if [ "$policy" = "$WL_INGRESS_POLICY" ]; then expected_uid=$HELD_POLICY_UID; fi+ kctl get networkpolicy "$policy" -n "$SANDBOX_NS" -o json | jq -e \+ --arg run "$E2E_RUN_ID" --arg policy "$policy" --arg held "$WL_INGRESS_POLICY" --arg uid "$expected_uid" '+ .metadata.labels["e2e.invalid/external-label"] == $run and+ .metadata.annotations["e2e.invalid/external-annotation"] == $run and+ .metadata.uid == $uid and+ (if $policy == $held then+ ((.metadata.finalizers // []) | index("e2e.invalid/hold-deletion") != null)+ else true end)' >/dev/null || {+ fail 'Ingress contraction discarded external metadata, finalizer, or policy identity'+ exit 1+ }+done+release_policy_hold+ok 'Ingress contraction removed extra permissions while preserving external metadata and finalizer'+wrc_assert_http_allowed 'UI route survives ingress contraction' "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 'api-ok'+wrc_assert_http_allowed 'Sibling route survives ingress contraction' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" 'db-ok'++header "Phase 6 — terminating race self-heals without another parent event"terminating_uid="$(wrc_np_uid "$SANDBOX_NS" "$WL_INGRESS_POLICY")"HELD_POLICY_UID=$terminating_uidterminating_generation="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')"@@ -330,7 +378,7 @@ wrc_wait_for_np_recreated "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$terminating_uid"wait_for_recipe_active 120wrc_assert_http_allowed "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"-header "Phase 6 — steady-state no-churn"+header "Phase 7 — steady-state no-churn"wrc_begin_np_observationwrc_track_np "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingresswrc_track_np "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter@@ -338,7 +386,7 @@ wrc_track_np "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingresswrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120-header "Phase 7 — cleanup"+header "Phase 8 — cleanup"cleanupCREATED=0for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; dodiff --git a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh b/scripts/tests/test-wrc-networkpolicy-probe-contract.shindex 7883ba4db..32d724c7e 100755--- a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh+++ b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh@@ -215,6 +215,9 @@ observe() {write-after)log_event '2026-09-04T00:00:03.000000000Z' "$noop_event"log_event '2026-09-04T00:00:03.100000000Z' "$write_event" ;;+ historical-write)+ log_event '2026-09-04T00:00:01.999999999Z' "$write_event"+ log_event '2026-09-04T00:00:03.000000000Z' "$noop_event" ;;wrong-family)log_event '2026-09-04T00:00:03.000000000Z' "${noop_event/workload-ingress/oauth-broker-egress}" ;;missing-witness) ;;@@ -230,10 +233,39 @@ observe() {}expect_pass 'complete no-write window with an exact active reconcile and stable observer/UID/RV' observe clean-for scenario in write-before write-after delayed-write wrong-family missing-witness old-logs observer-changed pretrigger-rv-drift; do+for scenario in wrong-family missing-witness old-logs observer-changed pretrigger-rv-drift; doexpect_reject "$scenario despite a completed parent reconcile" observe "$scenario"done+# Exercise every logged final-apply operation and both pre-DNS contraction+# actions. Keep UID/RV constant deliberately: log evidence must independently+# reject writes even when an API snapshot does not reveal them. The ordinary+# writer's wrong family and the contraction writer's absent family are real+# reasons to bind mutation events by namespace/name, not by family.+for operation in created replaced 'create failed' 'replace failed' 'read failed' read-retry contraction-replace contraction-delete; do+ case "$operation" in+ read-retry)+ write_event='{"msg":"network policy read unavailable; retrying once","namespace":"sandbox-recipes","policy":"policy","family":"workload-ingress"}' ;;+ contraction-*)+ write_event="$(jq -cn --arg action "${operation#contraction-}" \+ '{msg:"contracted recipe network policy",namespace:"sandbox-recipes",policy:"policy",action:$action}')" ;;+ *)+ write_event="$(jq -cn --arg msg "network policy $operation" \+ '{msg:$msg,namespace:"sandbox-recipes",policy:"policy",family:"wrong-family"}')" ;;+ esac+ tracked_write_event=$write_event+ for scenario in write-before write-after delayed-write; do+ expect_reject "$operation $scenario despite no-op and stable UID/RV" observe "$scenario"+ done+ expect_pass "$operation before observation cursor is excluded" observe historical-write+ for binding in namespace policy; do+ write_event="$(printf '%s' "$tracked_write_event" | jq -c --arg binding "$binding" '.[$binding] = "unrelated"')"+ for scenario in write-before write-after delayed-write; do+ expect_pass "$operation with unrelated $binding $scenario does not contaminate tracked policy" observe "$scenario"+ done+ done+done+# Exercise the actual OAuth caller sequence as well as the shared assertions.# A healthy authorised workload cannot substitute for the negative source's# post-control when that source has an independent persistent route failure.diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex a6a434abc..85a36e10b 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.347",+ "version": "0.1.348","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.347",+ "version": "0.1.348","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex a00abeb80..13e1eff8c 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.347",+ "version": "0.1.348","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.tsindex d37c830fd..75f7a09ad 100644--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts@@ -269,6 +269,15 @@ function gatewayIdentityLabelsMatch()}+/** Label-owned policies must not acquire another controller's GC authority. */+export function classifyOwnerlessNetworkPolicyOwnership(+ existing: k8s.V1NetworkPolicy+): NetworkPolicyOwnershipDecision {+ return (existing.metadata?.ownerReferences?.length ?? 0) === 0+ ? { kind: 'owned' }+ : { kind: 'conflict', reason: 'owner-reference-mismatch' }+}+export function classifyNetworkPolicyOwnership(family: NetworkPolicyFamily,desired: k8s.V1NetworkPolicy,@@ -277,9 +286,7 @@ export function classifyNetworkPolicyOwnership(const desiredOwners = controllerOwners(desired)const existingOwners = controllerOwners(existing)if (desiredOwners.length === 0) {- return (existing.metadata?.ownerReferences?.length ?? 0) === 0- ? { kind: 'owned' }- : { kind: 'conflict', reason: 'owner-reference-mismatch' }+ return classifyOwnerlessNetworkPolicyOwnership(existing)}const desiredOwner = desiredOwners[0]diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 5e5bd0395..2521fc95f 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -16,6 +16,8 @@ import type { NetworkPolicyFamily } from './networkPolicyConvergence'import * as brokerIssuer from './oauthBrokerTokenIssuerClient'import { PLUGIN_WORKLOAD_SDK_PROVIDER_UNAVAILABLE_CONDITION_TYPE } from './pluginWorkloadSdkValidator'import {+ buildUiIngressNetworkPolicy,+ buildWorkloadIngressNetworkPolicy,resolveResourceName,resolveScopedStatefulSetResourceName,resolveScopedWorkloadResourceName,@@ -508,6 +510,177 @@ describe('WorkflowRecipeReconciler', () => {})describe('#567 regression: policy contraction', () => {+ describe('#580 contraction lifecycle regression', () => {+ const namespace = 'sandbox-recipes'+ type Pipeline = {+ applyNetworkPolicyContraction(input: {+ existing: k8s.V1NetworkPolicy+ desired: k8s.V1NetworkPolicy | null+ namespace: string+ recipeName: string+ }): Promise<k8s.V1NetworkPolicy | null>+ applyNetworkPolicy(+ policy: k8s.V1NetworkPolicy,+ namespace: string,+ options: {+ family: NetworkPolicyFamily+ recipeName: string+ existing: k8s.V1NetworkPolicy | null+ }+ ): Promise<void>+ }+ for (const family of ['ui-ingress', 'workload-ingress'] as const) {+ const build = (ports: number[]) => {+ const recipe = makeRecipe()+ return (+ family === 'ui-ingress'+ ? buildUiIngressNetworkPolicy(recipe, 'app', ports, namespace, 'sandbox-ui')+ : buildWorkloadIngressNetworkPolicy(+ recipe.spec.workloads![0],+ recipe,+ namespace,+ ports.map(port => ({+ fromWorkloadId: 'source',+ fromNamespace: namespace,+ port,+ protocol: 'TCP' as const,+ }))+ )+ )!+ }+ const decorated = (ports = [443, 8443]) => {+ const policy = build(ports)+ policy.metadata!.labels!['external.example/team'] = 'payments'+ policy.metadata!.annotations = { 'external.example/review': 'retained' }+ policy.metadata!.finalizers = ['external.example/cleanup']+ return policy+ }+ it(`${family}: preserves external metadata through contraction and final apply`, async () => {+ const original = decorated()+ const live = policyStore([original])+ const name = original.metadata!.name!+ const desired = build([443])+ const pipeline = reconciler as unknown as Pipeline+ const contracted = await pipeline.applyNetworkPolicyContraction({+ existing: structuredClone(live.get(name)!),+ desired,+ namespace,+ recipeName: 'test-recipe',+ })+ await pipeline.applyNetworkPolicy(desired, namespace, {+ family,+ recipeName: 'test-recipe',+ existing: contracted,+ })+ expect(live.get(name)?.metadata).toMatchObject({+ labels: { 'external.example/team': 'payments' },+ annotations: { 'external.example/review': 'retained' },+ finalizers: ['external.example/cleanup'],+ })+ expect(live.get(name)?.spec).toEqual(desired.spec)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ })+ it(`${family}: keeps owned metadata on a no-op and fences an authorized delete`, async () => {+ const original = decorated([443])+ const live = policyStore([original])+ const existing = structuredClone(live.get(original.metadata!.name!)!)+ const pipeline = reconciler as unknown as Pipeline+ const input = { existing, namespace, recipeName: 'test-recipe' }+ expect(+ await pipeline.applyNetworkPolicyContraction({ ...input, desired: build([443]) })+ ).toEqual(existing)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ await pipeline.applyNetworkPolicyContraction({ ...input, desired: null })+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).toHaveBeenCalledWith({+ name: original.metadata!.name,+ namespace,+ body: {+ preconditions: {+ uid: existing.metadata!.uid,+ resourceVersion: existing.metadata!.resourceVersion,+ },+ },+ })+ })+ for (const lifecycle of ['foreign-owner', 'terminating'] as const) {+ it(`${family}: does not revoke an unsafe ${lifecycle} read-back`, async () => {+ const original = decorated()+ const live = policyStore([original])+ const name = original.metadata!.name!+ const existing = structuredClone(live.get(name)!)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async () => {+ const readBack = structuredClone(live.get(name)!)+ // Admission/concurrency re-broadens the spec; the defensive+ // deletion must not override a new lifecycle owner or termination.+ readBack.spec = original.spec+ if (lifecycle === 'terminating') readBack.metadata!.deletionTimestamp = new Date()+ else+ readBack.metadata!.ownerReferences = [+ {+ apiVersion: 'v1',+ kind: 'ConfigMap',+ name: 'foreign',+ uid: 'foreign-uid',+ controller: true,+ },+ ]+ return readBack+ })+ await expect(+ (reconciler as unknown as Pipeline).applyNetworkPolicyContraction({+ existing,+ desired: build([443]),+ namespace,+ recipeName: 'test-recipe',+ })+ ).rejects.toThrow(+ lifecycle === 'terminating' ? 'terminating' : 'owner-reference-mismatch'+ )+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+ }+ for (const operation of ['replace', 'delete', 'unchanged'] as const) {+ for (const lifecycle of [+ 'controller-owner',+ 'non-controller-owner',+ 'terminating',+ ] as const) {+ it(`${family}: vetoes ${lifecycle} before contraction ${operation}`, async () => {+ const original = decorated()+ if (lifecycle === 'terminating') original.metadata!.deletionTimestamp = new Date()+ else+ original.metadata!.ownerReferences = [+ {+ apiVersion: 'v1',+ kind: 'ConfigMap',+ name: 'foreign',+ uid: 'foreign-uid',+ controller: lifecycle === 'controller-owner',+ },+ ]+ const live = policyStore([original])+ await expect(+ (reconciler as unknown as Pipeline).applyNetworkPolicyContraction({+ existing: live.get(original.metadata!.name!)!,+ namespace,+ recipeName: 'test-recipe',+ desired:+ operation === 'delete'+ ? null+ : build(operation === 'replace' ? [443] : [443, 8443]),+ })+ ).rejects.toThrow(+ lifecycle === 'terminating' ? 'terminating' : 'owner-reference-mismatch'+ )+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()+ expect(mockNetworkingApi.deleteNamespacedNetworkPolicy).not.toHaveBeenCalled()+ })+ }+ }+ }+ })function policyStore(policies: k8s.V1NetworkPolicy[]) {const live = new Map<string, k8s.V1NetworkPolicy>()let version = 0diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex d0e2d294e..27c6d9100 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -107,6 +107,7 @@ import {import {type NetworkPolicyFamily,buildNetworkPolicyReplacement,+ classifyOwnerlessNetworkPolicyOwnership,decideNetworkPolicyConvergence,networkPolicyMetadataMatchesDesired,} from './networkPolicyConvergence'@@ -5296,7 +5297,24 @@ export class WorkflowRecipeReconciler {if (!policyName || !resourceVersion) {throw new Error('NetworkPolicy contraction requires policy name and resourceVersion')}- this.assertRecipeNetworkPolicyOwnership(existing, recipeName, policyName, namespace)+ const assertMutableSnapshot = (snapshot: k8s.V1NetworkPolicy): void => {+ // These recipe-scoped policies are label-owned, not garbage-collection+ // owned. Apply the same lifecycle veto as final convergence before even+ // a no-op/delete, and again before trusting or revoking the read-back.+ if (snapshot.metadata?.deletionTimestamp) {+ throw new RetryableReconcileError(+ `NetworkPolicy "${policyName}" in ${namespace} is terminating; retrying after deletion`+ )+ }+ this.assertRecipeNetworkPolicyOwnership(snapshot, recipeName, policyName, namespace)+ const ownership = classifyOwnerlessNetworkPolicyOwnership(snapshot)+ if (ownership.kind === 'conflict') {+ throw new NetworkPolicyOwnershipConflictError(+ `Refusing to mutate NetworkPolicy "${policyName}" in ${namespace}: ${ownership.reason}`+ )+ }+ }+ assertMutableSnapshot(existing)if (desired === null) {const uid = existing.metadata?.uid@@ -5321,12 +5339,12 @@ export class WorkflowRecipeReconciler {}if (!contractionWriteNeeded(existing, desired)) return existing- desired.metadata = { ...(desired.metadata ?? {}), resourceVersion }+ const replacement = buildNetworkPolicyReplacement(desired, existing)try {await this.networkingApi.replaceNamespacedNetworkPolicy({name: policyName,namespace,- body: desired,+ body: replacement,})} catch (error: unknown) {throwNetworkPolicyMutationError(error, policyName, namespace)@@ -5341,7 +5359,7 @@ export class WorkflowRecipeReconciler {if (getErrorCode(error) === 404) return nullthrow error}- this.assertRecipeNetworkPolicyOwnership(live, recipeName, policyName, namespace)+ assertMutableSnapshot(live)const liveUid = live.metadata?.uidconst liveResourceVersion = live.metadata?.resourceVersionif (