Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): harden NetworkPolicy recovery and validation
Alfredo Lopez475cdbfe0094
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 28f1c64e3..36a43ec50 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -546,7 +546,7 @@ jobs:run: bash scripts/tests/test-e2e-vitest-node-unit-gate.sh- name: Test WRC NetworkPolicy live-convergence E2E contract- run: bash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh+ run: make test-wrc-networkpolicy-contracts- name: Test GFS real-Postgres Minikube runner contractrun: bash scripts/tests/test-gfs-real-pg-minikube-gate.shdiff --git a/Makefile b/Makefileindex 6813b3f68..3f30839ec 100644--- a/Makefile+++ b/Makefile@@ -1083,7 +1083,7 @@ test-contracts: test-e2e-deps ## Run contract tests onlytest-e2e-bash: ## Run bash-based E2E suites (scripts/e2e/*.sh)@echo "Running bash E2E suites..."KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-workflow-runtime-gate.sh- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.shKUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-workflow-backend-compat.shKUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-sfs-legacy-job-cleanup.shKUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-sfs-security.sh@@ -1096,12 +1096,18 @@ test-e2e-workflow-runtime: ## Run workflow runtime E2E gate.PHONY: test-e2e-wrc-internal-dependency-networkpolicytest-e2e-wrc-internal-dependency-networkpolicy: ## Run issue #485 WRC internal-dependency NetworkPolicy E2E gate@echo "Running WRC internal-dependency NetworkPolicy E2E gate..."- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh.PHONY: test-e2e-wrc-networkpolicy-live-convergencetest-e2e-wrc-networkpolicy-live-convergence: ## Run all PR #580 WRC NetworkPolicy family, route, repair, owner, and no-churn E2E gates@echo "Running WRC NetworkPolicy live-convergence E2E gate..."- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh++.PHONY: test-wrc-networkpolicy-contracts+test-wrc-networkpolicy-contracts: ## Run hermetic WRC NetworkPolicy runner, probe, observation, and fixture contracts+ bash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh+ bash scripts/tests/test-wrc-networkpolicy-probe-contract.sh+ bash scripts/tests/test-wrc-fixtures.sh.PHONY: test-e2e-codex-subscription-network-boundarytest-e2e-codex-subscription-network-boundary: ## Codex LLM proxy NetworkPolicy boundary (exit 3 before deploy)diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.mdindex 0a123f28d..80273e931 100644--- a/docs/deploy/workflow-recipes-guide.md+++ b/docs/deploy/workflow-recipes-guide.md@@ -329,6 +329,9 @@ it cannot hide later drift.WRC logger. In steady state expect `network policy unchanged; skipping update` or`network policy egress set unchanged; skipping live apply` with `policy`, `namespace`and `family` fields. Re-baseline alerts keyed on the old `Updated …` console line.+ `recipe reconciliation completed` records recipe UID, generation, outcome and retry+ delay only after the reconciliation result is ready; entering a reconcile is not its+ completion signal.- **NetworkPolicy drift is auto-reverted.** A live change to `podSelector`,`policyTypes`, `ingress`, `egress`, WRC-authored metadata or the gateway controllerowner forces one optimistic-concurrency replace using the `resourceVersion` from the@@ -338,6 +341,11 @@ it cannot hide later drift.whose winner disappears, missing live `resourceVersion`, replace 404/409 or transientAPI failure re-enqueues the parent with bounded backoff. A status-only update is nottreated as the retry trigger.+- **Metadata ownership applies to writes too.** Replacing a policy preserves external+ labels, annotations and finalizers from the validated snapshot. Desired WRC keys win;+ retired egress state and the legacy spec-hash are removed. Additional foreign lifecycle+ owners are rejected, not silently dropped or adopted. The workload-egress pre-read uses+ the same transient-error classification as the downstream apply path.- **Other spec-hash-managed objects retain their existing drift contract.** For example,a manually scaled recipe Deployment is not reverted until the desired manifest changes;its replica-health drift remains visible in recipe status.@@ -352,7 +360,8 @@ it cannot hide later drift.### WRC NetworkPolicy live-convergence E2ENetworkPolicy changes must run the aggregate gate against an isolated development-cluster with an explicit context:+cluster with an explicit context. Its Make target acquires the existing profile mutation+lease; individual suites require that same inherited lease:```bashE2E_KUBECONTEXT=<branch-owned-context> \@@ -363,10 +372,11 @@ The aggregate fails unless all four executable suites run. Together they cover eNetworkPolicy family routed through the live-convergence helper:- `ui-ingress`, `workload-ingress` and `workload-egress`: correctly configured UI→API- and API→sibling traffic, unlabelled negative controls, forced live-spec drift, a real+ and API→sibling traffic, isolated ingress and undeclared-sibling egress controls,+ forced live-spec drift, a realparent recipe reconcile, repair, a terminating-policy race that self-heals through thescheduled retry without a second parent event, and a post-repair no-op witness plus- resourceVersion stability window;+ pre-trigger UID/resourceVersion baseline and full observation window;- `internal-dependency`: inferred source→backend traffic, an unlabelled denied pod,two-sided policy repair and no-churn;- `oauth-broker-egress`: opted-in workload→Control API reachability, an unlabelled@@ -374,13 +384,30 @@ NetworkPolicy family routed through the live-convergence helper:- `webhook-gateway`: all three policies, a signed public webhook business signal,spec drift, stale/missing owner repair, route recovery and no-churn.-Every no-churn assertion requires both a policy/family-bound structured WRC no-op log and-an unchanged `resourceVersion`; observing only the generic parent-reconcile log is not-sufficient. The public CI runner cannot provide this Calico-backed cluster. It therefore-runs the fail-loud composition/syntax contract, while the real dataplane result is-recorded by the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a-missing family suite, incomplete cleanup or zero executed suites are failures, never-skips.+Negative controls first prove reachability from the same probe with a narrow temporary+permission, remove only that permission, observe a remotely executed connection timeout,+and prove reachability again. Complementary permissions isolate ingress from egress.+An exec, authorization, missing-tool or malformed-response error fails the test instead+of counting as a policy denial. Condition-based polling accommodates CNI propagation.++Every no-churn observation begins before the parent trigger. It requires the exact+UID/generation completion receipt, every policy/family no-op witness, unchanged policy+UID/resourceVersion, stable controller replica/container identities, and no observed+policy write during the entire window. A no-op followed by a write fails even when the+resourceVersion does not change. Exact API-call counts are asserted separately in unit+tests; the runtime logs are not a substitute for exhaustive Kubernetes audit accounting.++Fixtures use unique run IDs, collision-safe creation, UID-preconditioned deletion and+owned dynamic port-forwards. Cleanup never enrolls an existing sample by name, and a+cleanup failure fails the suite. `E2E_KEEP_RESOURCES=1` explicitly retains run+resources for inspection, while still stopping owned port-forwards.++The public CI runner cannot provide this Calico-backed cluster. It runs+`make test-wrc-networkpolicy-contracts`: the actual aggregate against instrumented child+processes, every child failure/missing-child case, helper fault injection, and fixture+ownership/cleanup tests. Skipping the child invocation or swallowing a child error fails+this contract. The real dataplane result remains a separate local T2/pre-merge evidence+lane. Missing prerequisites, incomplete observations and zero execution are failures.### Unit tests (vitest + @testing-library/react)diff --git a/scripts/e2e/_lib/wrc-fixtures.sh b/scripts/e2e/_lib/wrc-fixtures.shnew file mode 100755index 000000000..36724e6ac--- /dev/null+++ b/scripts/e2e/_lib/wrc-fixtures.sh@@ -0,0 +1,203 @@+#!/usr/bin/env bash+# Run-owned WRC fixtures. The caller must first enter the branch mutation lease.+# shellcheck disable=SC1091+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../minikube" && pwd)/port-forward-owner.sh"++wrc_fixture_init() {+ wrc_require_networkpolicy_lease || return 1+ command -v jq >/dev/null || return 1+ E2E_RUN_ID="${E2E_RUN_ID:-$(openssl rand -hex 6)}"+ # Reject lossy normalization: two supplied IDs must never alias after trimming.+ [[ "$E2E_RUN_ID" =~ ^[a-z0-9]{1,12}$ ]] || {+ echo 'E2E_RUN_ID must contain 1–12 lowercase letters or digits' >&2; return 1;+ }+ export E2E_RUN_ID+ local root+ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"+ mkdir -p "$root/.local-notes/infra/runs"+ WRC_FIXTURE_DIR="$(mktemp -d "$root/.local-notes/infra/runs/wrc-${E2E_RUN_ID}-XXXXXX")"+ chmod 700 "$WRC_FIXTURE_DIR"+ WRC_FIXTURE_LEDGER="$WRC_FIXTURE_DIR/owned.jsonl"+ (umask 077; : > "$WRC_FIXTURE_LEDGER")+ jq -n --arg run "$E2E_RUN_ID" --arg profile "$MINIKUBE_PROFILE" \+ --arg context "$E2E_KUBECONTEXT" --arg worktree "$root" \+ '{run:$run,profile:$profile,context:$context,worktree:$worktree}' > "$WRC_FIXTURE_DIR/binding.json"+ WRC_PORT_FORWARD_PID=""+ log "WRC fixtures E2E_RUN_ID=${E2E_RUN_ID}; evidence=${WRC_FIXTURE_DIR}"+}++wrc_record_owned() {+ jq -ce '{apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace,+ uid:.metadata.uid,labels:(.metadata.labels // {})}} |+ select(.metadata.uid != null and .metadata.uid != "")' >> "$WRC_FIXTURE_LEDGER"+}++wrc_create_owned() {+ local created+ # create, never apply: a collision cannot change or enroll another run's object.+ created="$(jq --arg run "$E2E_RUN_ID" '.metadata.labels["e2e.clerum.io/run"]=$run' |+ kctl create --request-timeout=30s -f - -o json)" || return 1+ printf '%s\n' "$created" | wrc_record_owned+}++wrc_delete_owned() {+ local ns=$1 kind=$2 name=$3 wait_for_delete=${4:-true} owned current uid api_path+ [[ "$wait_for_delete" == true || "$wait_for_delete" == false ]] || return 1+ [[ "$ns" =~ ^[a-z0-9][a-z0-9.-]*$ && "$name" =~ ^[a-z0-9][a-z0-9.-]*$ ]] || return 1+ owned="$(jq -sc --arg ns "$ns" --arg kind "$kind" --arg name "$name" '+ [.[] | select(.metadata.namespace==$ns and (.kind|ascii_downcase)==($kind|ascii_downcase)+ and .metadata.name==$name)] | last // empty' "$WRC_FIXTURE_LEDGER")" || return 1+ [[ -n "$owned" ]] || { echo "Refusing unowned delete: $ns/$kind/$name" >&2; return 1; }+ current="$(kctl get "$kind" "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1+ [[ -n "$current" ]] || return 0+ uid="$(jq -r '.metadata.uid' <<< "$owned")"+ # UID is the immutable ownership identity. Additional controller labels are+ # allowed, but every label recorded at enrollment must still match.+ jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$current" >/dev/null || {+ echo "Ownership changed: $ns/$kind/$name" >&2; return 1;+ }+ # Use the native DELETE body to enforce UID atomically at the API server;+ # GET followed by a name-only delete could race Kubernetes GC/recreation.+ case "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" in+ workflowrecipe) api_path="/apis/clerum.io/v1alpha1/namespaces/$ns/workflowrecipes/$name" ;;+ networkpolicy) api_path="/apis/networking.k8s.io/v1/namespaces/$ns/networkpolicies/$name" ;;+ deployment) api_path="/apis/apps/v1/namespaces/$ns/deployments/$name" ;;+ pod|service|configmap|secret) api_path="/api/v1/namespaces/$ns/$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')s/$name" ;;+ *) echo "Unsupported owned resource kind: $kind" >&2; return 1 ;;+ esac+ jq -n --arg uid "$uid" '{apiVersion:"v1",kind:"DeleteOptions",+ preconditions:{uid:$uid},propagationPolicy:"Background"}' > "$WRC_FIXTURE_DIR/delete-options.json" || return 1+ kctl delete --raw "$api_path" --request-timeout=30s -f "$WRC_FIXTURE_DIR/delete-options.json" >/dev/null || return 1+ [[ "$wait_for_delete" == true ]] || return 0+ kctl wait --for=delete "$kind/$name" -n "$ns" --request-timeout=30s --timeout="${TIMEOUT_DELETE:-90}s" >/dev/null || return 1+ current="$(kctl get "$kind" "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o name)" || return 1+ [[ -z "$current" ]]+}++wrc_capture_recipe_children() {+ local ns=$1 name=$2 uid=$3 target objects+ for target in "$ns" sandbox-ui; do+ objects="$(kctl get deployment,service,configmap,networkpolicy -n "$target" --request-timeout=30s -o json)" || return 1+ # Cross-namespace UI resources cannot have recipe ownerReferences. WRC's+ # complete recipe namespace/name labels bind those to our collision-free CR.+ jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '.items[] |+ select(any(.metadata.ownerReferences[]?; .uid==$uid) or+ (.metadata.labels["clerum.io/recipe-namespace"]==$ns and+ .metadata.labels["clerum.io/recipe-name"]==$name) or+ (.metadata.namespace==$ns and+ .metadata.labels["clerum.io/managed-by"]=="workflow-recipes" and+ .metadata.labels["clerum.io/recipe"]==$name))' <<< "$objects" |+ while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done || return 1+ done+}++wrc_cleanup_owned() {+ local status=0 ns kind name uid+ # Capture generated resources before deleting recipes, including the UI+ # namespace where Kubernetes cannot garbage-collect a cross-namespace owner.+ while IFS=$'\t' read -r ns name uid; do+ [[ -n "$name" ]] || continue+ wrc_capture_recipe_children "$ns" "$name" "$uid" || status=1+ done < <(jq -r 'select(.kind=="WorkflowRecipe") | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' "$WRC_FIXTURE_LEDGER")+ # Recipes first stops reconciliation from recreating children during cleanup.+ while IFS=$'\t' read -r ns kind name; do+ [[ -n "$name" ]] || continue+ wrc_delete_owned "$ns" "$kind" "$name" || status=1+ done < <(jq -sr 'unique_by([.kind,.metadata.namespace,.metadata.name]) |+ sort_by(if .kind=="WorkflowRecipe" then 0 else 1 end)[] |+ [.metadata.namespace,.kind,.metadata.name]|@tsv' "$WRC_FIXTURE_LEDGER")+ return "$status"+}++wrc_assert_port_forward() {+ local actual_start command+ [[ -n "${WRC_PORT_FORWARD_PID:-}" ]] || return 1+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == live ]] || return 1+ actual_start="$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" || return 1+ [[ "$actual_start" == "$WRC_PORT_FORWARD_START" ]] || return 1+ command="$(pf_owner_process_command "$WRC_PORT_FORWARD_PID")" || return 1+ [[ "$command" == "$WRC_PORT_FORWARD_COMMAND" ]] || return 1+ [[ "$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" == "$WRC_PORT_FORWARD_START" ]]+}++wrc_port_forward_reported_port() {+ local remote=$1 port+ pf_owner_validate_port "$remote" || return 1+ port="$(sed -nE "s/^Forwarding from 127\\.0\\.0\\.1:([0-9]+) -> ${remote}$/\\1/p" "$WRC_FIXTURE_DIR/port-forward.log")" || return 1+ # Reject absent, duplicated, IPv6-only and out-of-range bindings. The local+ # port comes exclusively from this child's output, never from a shared URL.+ pf_owner_validate_port "$port" || return 1+ printf '%s\n' "$port"+}++wrc_start_port_forward() {+ local ns=$1 service=$2 remote=$3 deadline=$((SECONDS + 30)) command+ [[ -z "${WRC_PORT_FORWARD_PID:-}" && -n "${E2E_KUBECONTEXT:-}" && -n "${MINIKUBE_PROFILE:-}" ]] || return 1+ pf_owner_validate_port "$remote" || return 1+ WRC_PORT_FORWARD_PORT=""+ WRC_PORT_FORWARD_COMMAND="${KUBECTL_BIN} --context=${E2E_KUBECONTEXT} -n ${ns} port-forward --address=127.0.0.1 svc/${service} :${remote}"+ "$KUBECTL_BIN" "--context=${E2E_KUBECONTEXT}" -n "$ns" port-forward \+ --address=127.0.0.1 "svc/${service}" ":${remote}" >"$WRC_FIXTURE_DIR/port-forward.log" 2>&1 &+ WRC_PORT_FORWARD_PID=$!+ WRC_PORT_FORWARD_START="$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" || return 1+ # Capture the exact binary path returned by ps, whose first token may be an+ # absolute path even when KUBECTL_BIN was resolved through PATH.+ while (( SECONDS < deadline )); do+ command="$(pf_owner_process_command "$WRC_PORT_FORWARD_PID")" || return 1+ if [[ "${command#* }" == "${WRC_PORT_FORWARD_COMMAND#* }" && "${command%% *}" == */kubectl || "$command" == "$WRC_PORT_FORWARD_COMMAND" ]]; then+ WRC_PORT_FORWARD_COMMAND="$command"+ if WRC_PORT_FORWARD_PORT="$(wrc_port_forward_reported_port "$remote")"; then+ pf_owner_write_record_atomic "$WRC_FIXTURE_DIR/port-forward.pid" "$WRC_PORT_FORWARD_PID" \+ "$WRC_PORT_FORWARD_START" "$MINIKUBE_PROFILE" "$E2E_KUBECONTEXT" "$(pwd -P)" \+ "$ns" "$service" "$WRC_PORT_FORWARD_PORT" "$remote" || return 1+ wrc_assert_port_forward+ return+ fi+ fi+ sleep 0.1+ done+ echo 'Owned port-forward did not report a dynamic local binding' >&2+ return 1+}++wrc_reap_port_forward() {+ local record="$WRC_FIXTURE_DIR/port-forward.pid"+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]] || return 1+ # Startup can fail before publishing a record. If one exists, preserve it+ # unless both immutable process identity fields match this exact child.+ if [[ -e "$record" || -L "$record" ]]; then+ pf_owner_read_record "$record" || return 1+ [[ "$PF_OWNER_RECORD_PID" == "$WRC_PORT_FORWARD_PID" &&+ "$PF_OWNER_RECORD_START" == "$WRC_PORT_FORWARD_START" ]] || return 1+ fi+ pf_owner_reap_process "$WRC_PORT_FORWARD_PID"+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]] || return 1+ pf_owner_remove_dead_record "$record" || return 1+ WRC_PORT_FORWARD_PID=""+}++wrc_stop_port_forward() {+ [[ -n "${WRC_PORT_FORWARD_PID:-}" ]] || return 0+ local index+ # A dead/reused process is a conflict, including during successful cleanup.+ wrc_assert_port_forward || { echo 'Port-forward ownership/liveness conflict' >&2; return 1; }+ pf_owner_signal_process "$WRC_PORT_FORWARD_PID" TERM || return 1+ for ((index=0; index<40; index++)); do+ if [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]]; then+ wrc_reap_port_forward+ return+ fi+ if ! wrc_assert_port_forward; then+ # The child can exit between the state probe and ps, after our TERM.+ if [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]]; then+ wrc_reap_port_forward+ return+ fi+ return 1+ fi+ sleep 0.1+ done+ echo 'Owned port-forward did not exit within bounded cleanup' >&2+ return 1+}diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.shindex 9cc66bd83..1d9a78336 100755--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh@@ -4,7 +4,72 @@set -euo pipefail-WRC_NP_LAST_RECONCILE_SINCE_TIME=""+WRC_NP_RECONCILE_BINDING='{}'+WRC_NP_OBSERVATION_SINCE=""+WRC_NP_OBSERVATION_PODS='[]'+WRC_NP_TRACKED='[]'++wrc_require_networkpolicy_lease() {+ local root+ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"+ if [ -z "${KUBECONTEXT:-}" ] || [ "$KUBECONTEXT" != "${MINIKUBE_PROFILE:-}" ] ||+ [ "$KUBECONTEXT" != "${T2_CONTEXT:-${CONTROL_API_REAL_PG_CONTEXT:-}}" ]; then+ printf 'PROFILE_LOCK_REQUIRED: WRC NetworkPolicy E2E needs the matching explicit owned profile/context\n' >&2+ return 1+ fi+ bash "$root/scripts/minikube/require-t2-mutation-lock.sh"+}++wrc_now() {+ python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'+}++# Track every controller replica and container incarnation, so rolling/restarted+# observers cannot silently lose the beginning of a no-write window.+wrc_controller_snapshot() {+ local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}+ local deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes} definition selector replicas+ definition="$(kctl get deployment "$deployment" -n "$namespace" -o json)" || return 1+ selector="$(printf '%s' "$definition" | jq -er '+ .spec.selector | select((.matchExpressions // [] | length) == 0)+ | .matchLabels | select(length > 0) | to_entries | sort_by(.key)+ | map(.key + "=" + .value) | join(",")')" || return 1+ replicas="$(printf '%s' "$definition" | jq -er '.spec.replicas // 1 | select(. > 0)')" || return 1+ kctl get pods -n "$namespace" -l "$selector" -o json | jq -ce --argjson replicas "$replicas" '+ .items | select(length == $replicas)+ | select(all(.[]; .metadata.deletionTimestamp == null and .status.phase == "Running"+ and (.status.containerStatuses | length > 0)+ and all(.status.containerStatuses[]; .ready == true and (.containerID | length > 0))))+ | map({name:.metadata.name,uid:.metadata.uid,+ containers:(.status.containerStatuses | sort_by(.name) | map({name,containerID,restartCount}))})+ | sort_by(.name)'+}++wrc_controller_logs() {+ local snapshot=$1 since=$2 pod logs+ local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}+ while IFS= read -r pod; do+ [ -n "$pod" ] || return 1+ logs="$(kctl logs "$pod" -n "$namespace" --all-containers=true --timestamps=true \+ --since-time="$since")" || return 1+ printf '%s\n' "$logs"+ done < <(printf '%s' "$snapshot" | jq -r '.[].name')+}++# Kubernetes timestamps delimit the cursor; mixed legacy console lines are not+# JSON events. Rejecting writes is done over all events, never the first match.+wrc_log_events() {+ local since=$1+ jq -Rsc --arg since "$since" '+ def timestamp_key:+ sub("Z$"; "") | split(".")+ | .[0] + "." + (((.[1] // "") + "000000000")[0:9]);+ ($since | timestamp_key) as $start+ | split("\n") | map(+ try (capture("^(?<at>[0-9T:.+-]+Z) (?<payload>.*)$")+ | select((.at | timestamp_key) >= $start)+ | .payload | fromjson | select(type == "object")) catch empty)'+}wrc_np_spec_hash() {local namespace=$1 name=$2@@ -80,14 +145,12 @@ wrc_inject_selector_drift() {wrc_trigger_recipe_reconcile() {local namespace=$1 name=$2 timeout=${3:-120}- local before_generation after_generation marker patch since_time elapsed=0- local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}- local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}- local needle="[WR-Reconciler] Reconciling \"${name}\""+ local before_generation after_generation marker patch since_time started=$SECONDS+ local recipe uid snapshot events logs- before_generation="$(- kctl get workflowrecipe "$name" -n "$namespace" -o jsonpath='{.metadata.generation}'- )"+ recipe="$(kctl get workflowrecipe "$name" -n "$namespace" -o json)" || return 1+ before_generation="$(printf '%s' "$recipe" | jq -er '.metadata.generation')" || return 1+ uid="$(printf '%s' "$recipe" | jq -er '.metadata.uid | select(length > 0)')" || return 1[[ "$before_generation" =~ ^[0-9]+$ ]] || {fail "WorkflowRecipe ${namespace}/${name} has no numeric generation before reconcile trigger"return 1@@ -97,10 +160,8 @@ wrc_trigger_recipe_reconcile() {WRC_NP_RECONCILE_SEQUENCE=$((WRC_NP_RECONCILE_SEQUENCE + 1))marker="np-e2e-${WRC_NP_RECONCILE_SEQUENCE}-$(date +%s)"patch="$(jq -cn --arg description "$marker" '{spec:{description:$description}}')"- since_time="$(- python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'- )"- WRC_NP_LAST_RECONCILE_SINCE_TIME="$since_time"+ snapshot="$(wrc_controller_snapshot)" || return 1+ since_time="$(wrc_now)"kctl patch workflowrecipe "$name" -n "$namespace" --type=merge -p "$patch" >/dev/nullafter_generation="$(@@ -110,80 +171,213 @@ wrc_trigger_recipe_reconcile() {fail "WorkflowRecipe ${namespace}/${name} generation did not advance (${before_generation} -> ${after_generation})"return 1fi+ WRC_NP_RECONCILE_BINDING="$(jq -cn --arg recipe "$name" --arg namespace "$namespace" \+ --arg uid "$uid" --argjson generation "$after_generation" '{recipe:$recipe,namespace:$namespace,uid:$uid,generation:$generation}')"- while [ "$elapsed" -lt "$timeout" ]; do- if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \- --since-time="$since_time" 2>/dev/null | grep -Fq "$needle"; then- ok "WorkflowRecipe ${namespace}/${name} entered reconcile at generation ${after_generation}"+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do+ [ "$(wrc_controller_snapshot)" = "$snapshot" ] || {+ fail "WRC observer changed during the parent reconcile"+ return 1+ }+ logs="$(wrc_controller_logs "$snapshot" "$since_time")" || return 1+ events="$(printf '%s' "$logs" | wrc_log_events "$since_time")" || return 1+ if printf '%s' "$events" | jq -e --argjson binding "$WRC_NP_RECONCILE_BINDING" '+ any(.[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe+ and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation)' >/dev/null; then+ ok "WorkflowRecipe ${namespace}/${name} completed reconcile at generation ${after_generation}"return 0fisleep "$POLL_INTERVAL"- elapsed=$((elapsed + POLL_INTERVAL))donefail "Timed out waiting for WRC to reconcile WorkflowRecipe ${namespace}/${name} generation ${after_generation}"return 1}-wrc_wait_for_np_noop_witness() {- local namespace=$1 name=$2 family=$3 mode=${4:-apply} timeout=${5:-120} elapsed=0- local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}- local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}- local message='network policy unchanged; skipping update'+wrc_begin_np_observation() {+ WRC_NP_OBSERVATION_SINCE="$(wrc_now)"+ WRC_NP_OBSERVATION_PODS="$(wrc_controller_snapshot)" || return 1+ WRC_NP_TRACKED='[]'+ WRC_NP_RECONCILE_BINDING='{}'+}- [ -n "$WRC_NP_LAST_RECONCILE_SINCE_TIME" ] || {- fail "No parent reconcile timestamp is available for NetworkPolicy ${namespace}/${name}"+wrc_track_np() {+ local namespace=$1 name=$2 family=$3 mode=${4:-apply} policy+ local message='network policy unchanged; skipping update'+ [ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_RECONCILE_BINDING" = '{}' ] || {+ fail 'NetworkPolicy baseline must be captured before the reconcile trigger'return 1}if [ "$mode" = "workload-egress-prefilter" ]; thenmessage='network policy egress set unchanged; skipping live apply'elif [ "$mode" != "apply" ]; then- fail "Unknown NetworkPolicy no-op witness mode: ${mode}"+ fail "Unknown NetworkPolicy observation mode: ${mode}"return 1fi+ policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json)" || return 1+ policy="$(printf '%s' "$policy" | jq -ce --arg namespace "$namespace" --arg policy "$name" \+ --arg family "$family" --arg message "$message" '+ select(.metadata.deletionTimestamp == null and (.metadata.uid | length > 0)+ and (.metadata.resourceVersion | length > 0))+ | {namespace:$namespace,policy:$policy,family:$family,message:$message,+ uid:.metadata.uid,rv:.metadata.resourceVersion}')" || return 1+ WRC_NP_TRACKED="$(printf '%s' "$WRC_NP_TRACKED" | jq -c --argjson policy "$policy" '. + [$policy]')"+}- while [ "$elapsed" -lt "$timeout" ]; do- if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \- --since-time="$WRC_NP_LAST_RECONCILE_SINCE_TIME" 2>/dev/null |- jq -Rse --arg message "$message" --arg policy "$name" --arg namespace "$namespace" \- --arg family "$family" '- split("\n")- | map(try fromjson catch null)- | any(- . != null and- .msg == $message and- .policy == $policy and- .namespace == $namespace and- .family == $family- )- ' >/dev/null; then- ok "NetworkPolicy ${namespace}/${name} emitted its ${family} no-op witness"+wrc_assert_np_observation_clean() {+ local seconds=${1:-20} timeout=${2:-120} started=$SECONDS logs events row current missing+ [[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [[ "$timeout" =~ ^[1-9][0-9]*$ ]] &&+ [ "$seconds" -le 300 ] && [ "$timeout" -le 600 ] && [ "$timeout" -ge "$seconds" ] || return 1+ [ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_TRACKED" != '[]' ] &&+ [ "$WRC_NP_RECONCILE_BINDING" != '{}' ] || {+ fail 'NetworkPolicy observation has no baseline, tracked policies, or completed trigger'+ return 1+ }+ while [ "$((SECONDS - started))" -le "$timeout" ]; do+ [ "$(wrc_controller_snapshot)" = "$WRC_NP_OBSERVATION_PODS" ] || {+ fail 'WRC observer changed; the NetworkPolicy window is incomplete'+ return 1+ }+ logs="$(wrc_controller_logs "$WRC_NP_OBSERVATION_PODS" "$WRC_NP_OBSERVATION_SINCE")" || {+ fail 'Unable to read the complete WRC observation window'+ return 1+ }+ events="$(printf '%s' "$logs" | wrc_log_events "$WRC_NP_OBSERVATION_SINCE")" || return 1+ # Match writes by namespace/name even if a broken writer mislabels family.+ if printf '%s' "$events" | jq -e --argjson tracked "$WRC_NP_TRACKED" '+ any(.[]; . as $event | any($tracked[]; .namespace == $event.namespace and .policy == $event.policy)+ and ((.msg // "") | test("^network policy (created|replaced|create failed|replace failed|read failed)$")))' >/dev/null; then+ fail 'NetworkPolicy window contains a write or failed operation, even if a no-op also occurred'+ return 1+ fi+ while IFS= read -r row; do+ current="$(kctl get networkpolicy "$(printf '%s' "$row" | jq -r '.policy')" \+ -n "$(printf '%s' "$row" | jq -r '.namespace')" -o json)" || return 1+ if ! printf '%s' "$current" | jq -e --argjson before "$row" '+ .metadata.uid == $before.uid and .metadata.resourceVersion == $before.rv+ and .metadata.deletionTimestamp == null' >/dev/null; then+ fail 'NetworkPolicy changed since its pre-trigger baseline'+ return 1+ fi+ done < <(printf '%s' "$WRC_NP_TRACKED" | jq -c '.[]')+ missing="$(printf '%s' "$events" | jq -r --argjson tracked "$WRC_NP_TRACKED" \+ --argjson binding "$WRC_NP_RECONCILE_BINDING" '+ . as $events | ([ $tracked[] | . as $p | select(any($events[];+ .policy == $p.policy and .namespace == $p.namespace and .family == $p.family and .msg == $p.message) | not)] | length)+ + (if any($events[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe+ and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation+ and .phase == "active" and .requeueAfterMs == 0) then 0 else 1 end)')" || return 1+ if [ "$missing" = 0 ] && [ "$((SECONDS - started))" -ge "$seconds" ]; then+ ok 'Full NetworkPolicy window contains all no-op witnesses, no writes, and unchanged pre-trigger UID/resourceVersion'+ WRC_NP_OBSERVATION_SINCE=""return 0fisleep "$POLL_INTERVAL"- elapsed=$((elapsed + POLL_INTERVAL))done-- fail "NetworkPolicy ${namespace}/${name} emitted no ${family} no-op witness after the parent reconcile"+ fail 'NetworkPolicy observation lacked an exact completed reconcile or a required no-op witness'return 1}-wrc_assert_np_stable() {- local namespace=$1 name=$2 settle_seconds=${3:-20}- local before after- before="$(wrc_np_resource_version "$namespace" "$name")"- [ -n "$before" ] || {- fail "NetworkPolicy ${namespace}/${name} has no resourceVersion before stability window"- return 1- }- # This is an intentional no-churn observation window, not a readiness sleep:- # the policy is already converged and the assertion is that later reconciles- # do not update it.- sleep "$settle_seconds"- after="$(wrc_np_resource_version "$namespace" "$name")"- if [ "$after" != "$before" ]; then- fail "NetworkPolicy ${namespace}/${name} churned while converged (${before} -> ${after})"+# stdout carries a nonce-bound remote result; stderr/exec failure is a harness+# error. Only a completed connection timeout is a negative network observation.+wrc_http_probe() {+ local namespace=$1 target=$2 host=$3 port=$4 seconds=${E2E_CONNECT_TIMEOUT:-6}+ local nonce="np-$$-${RANDOM}-${RANDOM}" output begin end outcome encoded rest+ [[ "$port" =~ ^[0-9]+$ ]] && [ "$port" -gt 0 ] && [ "$port" -le 65535 ] &&+ [[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [ "$seconds" -le 30 ] || return 1+ # shellcheck disable=SC2016+ if ! output="$(kctl exec "$target" -n "$namespace" -- sh -c '+ export LC_ALL=C+ for tool in nc grep base64 tr mktemp; do command -v "$tool" >/dev/null 2>&1 || exit 20; done+ nonce=$1; host=$2; port=$3; seconds=$4+ errors=$(mktemp) || exit 21+ trap '\''rm -f "$errors"'\'' EXIT HUP INT TERM+ printf "WRC_NP_BEGIN|%s\n" "$nonce"+ response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$seconds" "$host" "$port" 2>"$errors")+ status=$?+ if [ "$status" -eq 0 ] && printf "%s" "$response" | grep -Eq "^HTTP/1\.[01] [2345][0-9][0-9]"; then+ printf "WRC_NP_END|%s|http|" "$nonce"+ printf "%s" "$response" | base64 | tr -d "\n"+ printf "\n"+ elif [ "$status" -ne 0 ] && [ -z "$response" ] && grep -Eqi "timed out|timeout" "$errors"; then+ printf "WRC_NP_END|%s|timeout|\n" "$nonce"+ else+ exit 22+ fi+ ' -- "$nonce" "$host" "$port" "$seconds")"; then+ fail "HTTP probe executor failed for ${namespace}/${target}; this is not NetworkPolicy denial" >&2return 1fi- ok "NetworkPolicy ${namespace}/${name} stayed at resourceVersion ${before}"+ begin="$(printf '%s\n' "$output" | sed -n '1p')"+ end="$(printf '%s\n' "$output" | sed -n '2p')"+ rest="$(printf '%s\n' "$output" | sed -n '3,$p')"+ [ "$begin" = "WRC_NP_BEGIN|$nonce" ] && [ -z "$rest" ] &&+ [[ "$end" == "WRC_NP_END|${nonce}|"* ]] || return 1+ end="${end#WRC_NP_END|${nonce}|}"+ outcome="${end%%|*}"+ encoded="${end#*|}"+ case "$outcome" in+ http) [ -n "$encoded" ] || return 1 ;;+ timeout) [ -z "$encoded" ] || return 1 ;;+ *) return 1 ;;+ esac+ jq -cen --arg outcome "$outcome" --arg body "$encoded" '+ {outcome:$outcome,body:$body}+ | select((.outcome == "timeout" and .body == "") or+ (.outcome == "http" and (.body | test("^[A-Za-z0-9+/]+={0,2}$"))+ and ((.body | length) % 4 == 0)+ and (.body | @base64d | test("^HTTP/1\\.[01] [2345][0-9][0-9]"))))'+}++wrc_assert_http_allowed() {+ local description=$1 namespace=$2 target=$3 host=$4 port=$5 expected=${6:-} result+ local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}+ [[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do+ result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1+ if printf '%s' "$result" | jq -e --arg expected "$expected" '+ .outcome == "http" and ((.body | @base64d) | contains($expected))' >/dev/null; then+ ok "$description"+ return 0+ fi+ # Policy acknowledgement precedes CNI programming. Only valid remote+ # network outcomes may be retried; executor failures abort immediately.+ sleep "$POLL_INTERVAL"+ done+ fail "$description: expected an HTTP response from the reachable fixture"+ return 1+}++wrc_assert_http_blocked() {+ local description=$1 namespace=$2 target=$3 host=$4 port=$5 result+ local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}+ [[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do+ result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1+ if printf '%s' "$result" | jq -e '.outcome == "timeout"' >/dev/null; then+ ok "$description"+ return 0+ fi+ sleep "$POLL_INTERVAL"+ done+ fail "$description: unexpected successful connection"+ return 1+}++# A fixture-owned permission for one direction, peer and port. Complementary+# controls isolate the boundary being tested without changing platform policy.+wrc_create_connection_policy() {+ local name=$1 direction=$2 namespace=$3 selector=$4 peer_namespace=$5 peer_selector=$6 port=$7+ case "$direction" in Ingress|Egress) ;; *) return 1 ;; esac+ jq -cn --arg name "$name" --arg direction "$direction" --arg namespace "$namespace" \+ --argjson selector "$selector" --arg peerNamespace "$peer_namespace" \+ --argjson peerSelector "$peer_selector" --argjson port "$port" '+ {namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":$peerNamespace}},+ podSelector:{matchLabels:$peerSelector}} as $peer+ | {ports:[{protocol:"TCP",port:$port}]} as $ports+ | {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",+ metadata:{name:$name,namespace:$namespace},+ spec:({podSelector:{matchLabels:$selector},policyTypes:[$direction]} ++ if $direction == "Ingress" then {ingress:[($ports + {from:[$peer]})]}+ else {egress:[($ports + {to:[$peer]})]} end)}' | wrc_create_owned}diff --git a/scripts/e2e/e2e-sandbox-ui-oauth.sh b/scripts/e2e/e2e-sandbox-ui-oauth.shindex d0a4cfeea..6f201905a 100755--- a/scripts/e2e/e2e-sandbox-ui-oauth.sh+++ b/scripts/e2e/e2e-sandbox-ui-oauth.sh@@ -31,22 +31,29 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${SCRIPT_DIR}/e2e-lib.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"++if [ "${1:-}" = "--cleanup-only" ]; then+ echo "Cleanup requires the current run ownership ledger; use normal EXIT cleanup." >&2+ exit 1+fi+wrc_fixture_initRECIPE_FILE="workflow-recipes/samples/sandbox-ui-oauth-hello.yaml"-RECIPE_NAME="sandbox-ui-oauth-hello"+RECIPE_NAME="e2e-oauth-${E2E_RUN_ID}"WORKLOAD_ID="hello"SANDBOX_UI_NS="sandbox-ui"-OAUTH_SECRET_NAME="slack-oauth-creds"-BACKGROUND_RECIPE_NAME="e2e-wrc-oauth-broker"+OAUTH_SECRET_NAME="e2e-oauth-secret-${E2E_RUN_ID}"+BACKGROUND_RECIPE_NAME="e2e-broker-${E2E_RUN_ID}"BACKGROUND_WORKLOAD_ID="background-worker"BACKGROUND_POLICY_NAME="wf-${BACKGROUND_RECIPE_NAME}-oauth-broker-egress"-BACKGROUND_PROBE_POD="e2e-wrc-oauth-unlabelled"+BACKGROUND_PROBE_POD="e2e-probe-${E2E_RUN_ID}"STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"TIMEOUT_RECIPE_ACTIVE="${TIMEOUT_RECIPE_ACTIVE:-180}"UI_DEPLOYMENT=""BACKGROUND_DEPLOYMENT=""-UI_PORT_FWD_PORT="${E2E_OAUTH_UI_PORT:-18096}"-UI_PORT_FWD_PID=""+# Local UI port is allocated by the owned kubectl child.CREATED=0wait_for_recipe_phase() {@@ -78,21 +85,18 @@ wait_for_workload_instance() {return 1}-http_response_from_target() {- local target=$1 host=$2 port=$3- # shellcheck disable=SC2016- kctl exec "$target" -n "$WORKFLOW_RECIPE_NS" -- sh -c 'printf "GET /health HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w 6 "$1" "$2"' -- "$host" "$port" 2>/dev/null || true-}-create_background_probe() {- cat <<YAML | kctl apply -f - >/dev/null+ # This label satisfies Control API's independent ingress policy. The pod+ # deliberately has no recipe/workload labels selected by OAuth egress.+ cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: v1kind: Podmetadata:name: ${BACKGROUND_PROBE_POD}namespace: ${WORKFLOW_RECIPE_NS}labels:- e2e.clerum.io/probe: wrc-oauth-unlabelled+ e2e.clerum.io/probe: ${BACKGROUND_PROBE_POD}+ clerum.io/oauth-broker-client: "true"spec:restartPolicy: NeversecurityContext:@@ -110,50 +114,13 @@ spec:capabilities:drop: ["ALL"]YAML- wait_for_pod "$WORKFLOW_RECIPE_NS" "e2e.clerum.io/probe=wrc-oauth-unlabelled" 60+ wait_for_pod "$WORKFLOW_RECIPE_NS" "e2e.clerum.io/probe=${BACKGROUND_PROBE_POD}" 60}cleanup() {local cleanup_status=0- header "Cleanup"- if [ -n "$UI_PORT_FWD_PID" ]; then- if kill -0 "$UI_PORT_FWD_PID" 2>/dev/null; then- kill "$UI_PORT_FWD_PID" 2>/dev/null || cleanup_status=1- fi- wait "$UI_PORT_FWD_PID" 2>/dev/null || true- UI_PORT_FWD_PID=""- fi- kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found \- --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_status=1- kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" \- "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" \- "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1- # WRC GC follows ownerReferences inside the recipe namespace, but UI- # workloads are reconciled cross-namespace (sandbox-recipes →- # sandbox-ui) so K8s GC can't follow the link. Force-delete leftovers.- if [ -n "$UI_DEPLOYMENT" ]; then- kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- fi- kctl delete deployment,service -n "$SANDBOX_UI_NS" -l "clerum.io/recipe=${RECIPE_NAME}" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl delete deployment,service,networkpolicy -n "$WORKFLOW_RECIPE_NS" \- -l "clerum.io/recipe=${BACKGROUND_RECIPE_NAME}" --ignore-not-found \- >/dev/null 2>&1 || cleanup_status=1- kctl delete networkpolicy "ui-egress-${RECIPE_NAME}" -n "$SANDBOX_UI_NS" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl delete secret "$OAUTH_SECRET_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1-+ wrc_stop_port_forward || cleanup_status=1+ wrc_cleanup_owned || cleanup_status=1return "$cleanup_status"}@@ -161,6 +128,7 @@ on_exit() {local status=$? cleanup_status=0trap - EXIT INT TERMif [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then+ wrc_stop_port_forward || status=1warn "E2E_KEEP_RESOURCES=1; preserving OAuth fixtures for inspection"exit "$status"fi@@ -178,10 +146,7 @@ on_exit() {exit "$status"}-if [[ "${1:-}" == "--cleanup-only" ]]; then- cleanup- exit $?-fi+# Resources are cleaned only through this process's ownership ledger.trap on_exit EXITtrap 'exit 130' INT@@ -191,8 +156,7 @@ trap 'exit 143' TERMcheck_prerequisites# ─── Phase 1: Clean Slate ────────────────────────────────────────────-header "Phase 1 — Clean Slate"-cleanup >/dev/null 2>&1 || true+header "Phase 1 — Isolated run ${E2E_RUN_ID}"CREATED=1# ─── Phase 2: Create dummy OAuth credentials Secret ──────────────────@@ -201,12 +165,17 @@ kctl create secret generic "$OAUTH_SECRET_NAME" \-n "$WORKFLOW_RECIPE_NS" \--from-literal=client-id="dummy-client-id-for-platform-test" \--from-literal=client-secret="dummy-client-secret-for-platform-test" \- >/dev/null+ --dry-run=client -o json | wrc_create_ownedok "Secret '${OAUTH_SECRET_NAME}' created in '${WORKFLOW_RECIPE_NS}'"# ─── Phase 3: Apply Recipe ───────────────────────────────────────────-apply_recipe "$RECIPE_FILE" "$RECIPE_NAME"-cat <<YAML | kctl apply -f - >/dev/null+kctl create --dry-run=client -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" -o json |+ jq --arg name "$RECIPE_NAME" --arg ns "$WORKFLOW_RECIPE_NS" --arg secret "$OAUTH_SECRET_NAME" '+ .metadata.name=$name | .metadata.namespace=$ns |+ .spec.oauthClients[].clientIdRef.name=$secret |+ .spec.oauthClients[].clientSecretRef.name=$secret' > "$WRC_FIXTURE_DIR/ui-recipe.json"+wrc_create_owned < "$WRC_FIXTURE_DIR/ui-recipe.json"+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: clerum.io/v1alpha1kind: WorkflowRecipemetadata:@@ -281,36 +250,53 @@ control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.sexit 1}-response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then- ok "Opted-in background workload reaches the OAuth broker network boundary"-else- fail "Opted-in background workload cannot reach the OAuth broker network boundary"- exit 1-fi+wrc_assert_http_allowed "Opted-in background workload reaches OAuth broker" \+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090create_background_probe-response="$(http_response_from_target "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090)"-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then- fail "Unlabelled sandbox workload reached Control API through the OAuth policy"- exit 1-fi-ok "Unlabelled sandbox workload cannot use the OAuth broker egress"+# Establish reachability from this exact pod before removing the one temporary+# egress permission. The negative then isolates OAuth egress, not API ingress.+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned+apiVersion: networking.k8s.io/v1+kind: NetworkPolicy+metadata:+ name: ${BACKGROUND_PROBE_POD}+ namespace: ${WORKFLOW_RECIPE_NS}+spec:+ podSelector:+ matchLabels:+ e2e.clerum.io/probe: ${BACKGROUND_PROBE_POD}+ policyTypes: [Egress]+ egress:+ - to:+ - namespaceSelector:+ matchLabels:+ kubernetes.io/metadata.name: ${CONTROL_NS}+ podSelector:+ matchLabels:+ app: control-api+ ports:+ - protocol: TCP+ port: 8090+YAML+wrc_assert_http_allowed "OAuth negative probe reaches the same broker with temporary egress" \+ "$WORKFLOW_RECIPE_NS" "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090+wrc_delete_owned "$WORKFLOW_RECIPE_NS" networkpolicy "$BACKGROUND_PROBE_POD"+wrc_assert_http_blocked "Probe without recipe/workload labels cannot use OAuth egress" \+ "$WORKFLOW_RECIPE_NS" "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090+wrc_assert_http_allowed "OAuth broker remains reachable after the negative" \+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090oauth_policy_hash="$(wrc_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME")"wrc_inject_selector_drift "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME"wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120wrc_wait_for_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$oauth_policy_hash" 120-response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then- ok "OAuth broker route recovered after live NetworkPolicy repair"-else- fail "OAuth broker route did not recover after live NetworkPolicy repair"- exit 1-fi+wrc_assert_http_allowed "OAuth broker route recovered after policy repair" \+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090+wrc_begin_np_observation+wrc_track_np "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress applywrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120-wrc_wait_for_np_noop_witness "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress apply 120-wrc_assert_np_stable "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$STABILITY_SECONDS"+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120# ─── Phase 5: spec.oauthClients[] survives the round-trip ────────────header "Phase 5 — spec.oauthClients[] preserved on the WorkflowRecipe"@@ -366,10 +352,8 @@ elsefail "Service '${UI_DEPLOYMENT}' not found in '${SANDBOX_UI_NS}'"fi-kctl port-forward -n "$SANDBOX_UI_NS" "service/${UI_DEPLOYMENT}" \- "${UI_PORT_FWD_PORT}:8080" >/dev/null 2>&1 &-UI_PORT_FWD_PID=$!-ui_base_url="http://127.0.0.1:${UI_PORT_FWD_PORT}"+wrc_start_port_forward "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" 8080+ui_base_url="http://127.0.0.1:${WRC_PORT_FORWARD_PORT}"elapsed=0while [ "$elapsed" -lt 30 ]; doif curl -fsS --max-time 2 "${ui_base_url}/" >/dev/null 2>&1; then@@ -378,11 +362,12 @@ while [ "$elapsed" -lt 30 ]; dosleep "$POLL_INTERVAL"elapsed=$((elapsed + POLL_INTERVAL))done-if ! kill -0 "$UI_PORT_FWD_PID" 2>/dev/null ||+if ! wrc_assert_port_forward ||! curl -fsS --max-time 2 "${ui_base_url}/" >/dev/null 2>&1; thenfail "Scoped OAuth UI Service did not become reachable through its port-forward"exit 1fi+wrc_assert_port_forwardok "Scoped OAuth UI Service serves its visible route"# ─── Phase 7: UI lifecycle assets serve correctly ────────────────────diff --git a/scripts/e2e/e2e-webhooks-basic.sh b/scripts/e2e/e2e-webhooks-basic.shindex bb736c2c9..92902b25c 100755--- a/scripts/e2e/e2e-webhooks-basic.sh+++ b/scripts/e2e/e2e-webhooks-basic.sh@@ -5,7 +5,7 @@# resources → public POST → verify → forward → handler responds 200.## Phases:-# 1. Clean slate (delete any leftover from prior runs).+# 1. Allocate isolated run fixtures and preserve sample resources.# 2. Create the signing-secret Secret in sandbox-recipes.# 3. Apply samples/webhook-hello.yaml.# 4. Wait for the per-recipe webhook-gateway Deployment to be Ready.@@ -31,20 +31,28 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${SCRIPT_DIR}/e2e-lib.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"++if [ "${1:-}" = "--cleanup-only" ]; then+ echo "Cleanup requires the current run ownership ledger; use normal EXIT cleanup." >&2+ exit 1+fi+wrc_fixture_init-require_safe_kube_context+# The fixture initializer requires the branch mutation lease.RECIPE_FILE="workflow-recipes/samples/webhook-hello.yaml"-RECIPE_NAME="webhook-hello"+RECIPE_NAME="e2e-hook-${E2E_RUN_ID}"WEBHOOK_ID="hello"-SECRET_NAME="webhook-hello-creds"+SECRET_NAME="e2e-hook-secret-${E2E_RUN_ID}"SECRET_KEY="signing-secret"SECRET_VALUE="e2e-fireflies-test-secret"GATEWAY_NAME="wf-${RECIPE_NAME}-webhook-gateway"GATEWAY_NS="sandbox-recipes"PROXY_NS="webhook-ingress"-PROXY_DEPLOYMENT="webhook-proxy"+PROXY_SERVICE="webhook-proxy"PROXY_PORT=8095STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"@@ -52,8 +60,7 @@ PROXY_INGRESS_POLICY="allow-webhook-proxy-ingress-wf-${RECIPE_NAME}"HANDLER_EGRESS_POLICY="allow-gateway-egress-to-handler-wf-${RECIPE_NAME}"HANDLER_INGRESS_POLICY="allow-gateway-ingress-to-handler-wf-${RECIPE_NAME}"-PORT_FWD_PORT=18095-PORT_FWD_PID=""+PORT_FWD_PORT=""HANDLER_DEPLOYMENT=""CREATED=0@@ -91,36 +98,8 @@ wait_for_recipe_active() {cleanup() {local cleanup_status=0- if [ -n "$PORT_FWD_PID" ]; then- if kill -0 "$PORT_FWD_PID" 2>/dev/null; then- kill "$PORT_FWD_PID" 2>/dev/null || cleanup_status=1- fi- wait "$PORT_FWD_PID" 2>/dev/null || true- PORT_FWD_PID=""- fi- kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" \- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1- wait_for_workflowrecipe_deleted "$GATEWAY_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \- >/dev/null 2>&1 || cleanup_status=1- if [ -n "$HANDLER_DEPLOYMENT" ]; then- kctl -n "$GATEWAY_NS" delete deployment "$HANDLER_DEPLOYMENT" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete service "$HANDLER_DEPLOYMENT" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- fi- kctl -n "$GATEWAY_NS" delete deployment,service -l "clerum.io/recipe=${RECIPE_NAME}" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete service "$GATEWAY_NAME" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete configmap "wf-${RECIPE_NAME}-webhook-gateway-config" \- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete networkpolicy "$PROXY_INGRESS_POLICY" \- "$HANDLER_EGRESS_POLICY" "$HANDLER_INGRESS_POLICY" --ignore-not-found \- >/dev/null 2>&1 || cleanup_status=1- kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found \- >/dev/null 2>&1 || cleanup_status=1+ wrc_stop_port_forward || cleanup_status=1+ wrc_cleanup_owned || cleanup_status=1return "$cleanup_status"}@@ -128,6 +107,7 @@ on_exit() {local status=$? cleanup_status=0trap - EXIT INT TERMif [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then+ wrc_stop_port_forward || status=1warn "E2E_KEEP_RESOURCES=1; preserving webhook fixtures for inspection"exit "$status"fi@@ -145,27 +125,20 @@ on_exit() {exit "$status"}-if [ "${1:-}" = "--cleanup-only" ]; then- cleanup- exit $?-fi+# Resources are cleaned only through this process's ownership ledger.trap on_exit EXITtrap 'exit 130' INTtrap 'exit 143' TERM# ─── Phase 1: clean slate ───────────────────────────────────────────-header "Phase 1 — clean slate"-cleanup >/dev/null 2>&1 || true+header "Phase 1 — Isolated run ${E2E_RUN_ID}"CREATED=1-log "leftover state cleared"# ─── Phase 2: create signing-secret ─────────────────────────────────header "Phase 2 — provision signing-secret Secret"if kctl -n "$GATEWAY_NS" create secret generic "$SECRET_NAME" \- --from-literal="${SECRET_KEY}=${SECRET_VALUE}" >/dev/null 2>&1; then- kctl -n "$GATEWAY_NS" label secret "$SECRET_NAME" \- "clerum.io/owner-recipe=${RECIPE_NAME}" --overwrite >/dev/null+ --from-literal="${SECRET_KEY}=${SECRET_VALUE}" --dry-run=client -o json | wrc_create_owned; thenok "Secret $SECRET_NAME/$SECRET_KEY created in $GATEWAY_NS"elsefail "Secret create failed"@@ -173,9 +146,13 @@ elsefi# ─── Phase 3: apply recipe ──────────────────────────────────────────-header "Phase 3 — apply WorkflowRecipe"-if kctl apply -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" >/dev/null; then- ok "WorkflowRecipe applied"+header "Phase 3 — create isolated WorkflowRecipe"+kctl create --dry-run=client -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" -o json |+ jq --arg name "$RECIPE_NAME" --arg ns "$GATEWAY_NS" --arg secret "$SECRET_NAME" '+ .metadata.name=$name | .metadata.namespace=$ns |+ .spec.webhooks[].verification.secretRef.name=$secret' > "$WRC_FIXTURE_DIR/webhook-recipe.json"+if wrc_create_owned < "$WRC_FIXTURE_DIR/webhook-recipe.json"; then+ ok "Isolated WorkflowRecipe created"elsefail "WorkflowRecipe apply failed"exit 1@@ -233,12 +210,11 @@ fi# ─── Phase 6: POST a signed payload via webhook-proxy ───────────────header "Phase 6 — port-forward webhook-proxy and POST signed payload"-kctl -n "$PROXY_NS" port-forward "deploy/${PROXY_DEPLOYMENT}" "${PORT_FWD_PORT}:${PROXY_PORT}" \- >/dev/null 2>&1 &-PORT_FWD_PID=$!+wrc_start_port_forward "$PROXY_NS" "$PROXY_SERVICE" "$PROXY_PORT"+PORT_FWD_PORT="$WRC_PORT_FORWARD_PORT"proxy_ready=0for _ in $(seq 1 30); do- kill -0 "$PORT_FWD_PID" 2>/dev/null || break+ wrc_assert_port_forward || breakif curl -sf --max-time 2 "http://127.0.0.1:${PORT_FWD_PORT}/healthz" >/dev/null; thenproxy_ready=1break@@ -246,6 +222,7 @@ for _ in $(seq 1 30); dosleep 1doneif [ "$proxy_ready" = "1" ]; then+ wrc_assert_port_forwardok "webhook-proxy /healthz reachable via port-forward"elsefail "webhook-proxy /healthz unreachable"@@ -265,12 +242,14 @@ SIG="sha256=$(sign "$SECRET_VALUE" "$PAYLOAD")"http_status() {local method=$1 path=$2 sig=${3:-} body=${4:-}- local args=(-s -o /dev/null -w '%{http_code}' -X "$method"+ wrc_assert_port_forward || return 1+ local args=(-s --max-time 15 -o "$WRC_FIXTURE_DIR/webhook-response" -w '%{http_code}' -X "$method""http://127.0.0.1:${PORT_FWD_PORT}${path}"-H "content-type: application/json")[ -n "$sig" ] && args+=(-H "x-hub-signature-256: ${sig}")[ -n "$body" ] && args+=(--data "$body")- curl "${args[@]}"+ curl "${args[@]}" || return 1+ wrc_assert_port_forward}wait_for_policy_owner_uid() {@@ -288,7 +267,7 @@ wait_for_policy_owner_uid() {}status=$(http_status POST "$PUBLIC_PATH" "$SIG" "$PAYLOAD")-if [ "$status" = "200" ]; then+if [ "$status" = "200" ] && [ "$(cat "$WRC_FIXTURE_DIR/webhook-response")" = "webhook-ok" ]; thenok "POST with valid signature → 200"elsefail "POST with valid signature → ${status} (expected 200)"@@ -310,7 +289,7 @@ fi# 7.2 Oversize body via Content-Length. Stream it so the fixture never places a# multi-megabyte value in argv (which exceeds ARG_MAX on macOS/Linux runners).-status=$(head -c 2000000 </dev/zero | tr '\0' 'x' | curl -s -o /dev/null -w '%{http_code}' \+status=$(head -c 2000000 </dev/zero | tr '\0' 'x' | curl -s --max-time 15 -o /dev/null -w '%{http_code}' \-X POST "http://127.0.0.1:${PORT_FWD_PORT}${PUBLIC_PATH}" \-H "content-type: application/json" \-H "content-length: 2000000" \@@ -374,24 +353,19 @@ wait_for_policy_owner_uid "$HANDLER_INGRESS_POLICY" "$recipe_uid" 120ok "Webhook gateway policies repaired spec drift, stale owner and missing owner"status=$(http_status POST "$PUBLIC_PATH" "$SIG" "$PAYLOAD")-if [ "$status" = "200" ]; then+if [ "$status" = "200" ] && [ "$(cat "$WRC_FIXTURE_DIR/webhook-response")" = "webhook-ok" ]; thenok "Signed webhook route recovered after policy repair"elsefail "Signed webhook route returned ${status} after policy repair"fi-proxy_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")"-handler_egress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")"-handler_ingress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")"+wrc_begin_np_observation+wrc_track_np "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply+wrc_track_np "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply+wrc_track_np "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway applywrc_trigger_recipe_reconcile "$GATEWAY_NS" "$RECIPE_NAME" 120-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply 120-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply 120-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway apply 120-sleep "$STABILITY_SECONDS"-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")" = "$proxy_rv" ] || fail "${PROXY_INGRESS_POLICY} churned"-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")" = "$handler_egress_rv" ] || fail "${HANDLER_EGRESS_POLICY} churned"-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")" = "$handler_ingress_rv" ] || fail "${HANDLER_INGRESS_POLICY} churned"-ok "All three webhook gateway policies stayed resourceVersion-stable"+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120+ok "All three webhook policies witnessed no-op and stable UID/resourceVersion across the observation"# ─── Summary ────────────────────────────────────────────────────────header "Summary"diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.shindex b7f186610..0b9f352a5 100755--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh@@ -9,57 +9,47 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${SCRIPT_DIR}/e2e-lib.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"-raw_run_id="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"-RUN_ID="$(printf "%s" "$raw_run_id" | tr '[:upper:]' '[:lower:]' \- | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g' | cut -c1-20)"-[ -n "$RUN_ID" ] || RUN_ID="run-$$"--RECIPE_NAME="${E2E_RECIPE_NAME:-e2e-wrc-intdep-${RUN_ID}}"-SOURCE_ID="${E2E_SOURCE_ID:-src-${RUN_ID}}"-BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"-DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"+[ "$#" -eq 0 ] || {+ fail 'This suite accepts no arguments; cleanup is restricted to resources created by this invocation'+ exit 2+}+wrc_fixture_init++RECIPE_NAME="e2e-intdep-${E2E_RUN_ID}"+SOURCE_ID="src-${E2E_RUN_ID}"+BACKEND_ID="be-${E2E_RUN_ID}"+DENIED_POD="deny-${E2E_RUN_ID}"+PROBE_EGRESS_POLICY="intdep-probe-egress-${E2E_RUN_ID}"+PROBE_INGRESS_POLICY="intdep-probe-ingress-${E2E_RUN_ID}"BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"-CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"SOURCE_DEPLOYMENT="$SOURCE_ID"BACKEND_DEPLOYMENT="$BACKEND_ID"-CREATED=0+CLEANUP_DONE=0cleanup() {- local status=0- kctl delete pod "$DENIED_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false \- >/dev/null 2>&1 || status=1- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found --wait=false >/dev/null 2>&1 || status=1- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \- >/dev/null 2>&1 || status=1- kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \- --ignore-not-found --wait=false >/dev/null 2>&1 || status=1- kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \- --ignore-not-found >/dev/null 2>&1 || status=1- kctl delete networkpolicy -n "$SANDBOX_NS" \- -l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \- --ignore-not-found >/dev/null 2>&1 || status=1- return "$status"+ wrc_cleanup_owned}on_exit() {local status=$?- if [ "$CREATED" = "1" ] && [ "${E2E_KEEP_RESOURCES:-0}" != "1" ]; then- cleanup >/dev/null 2>&1 || {- [ "$status" -ne 0 ] && warn "issue #485 E2E cleanup left resources behind"- [ "$status" -eq 0 ] && fail "issue #485 E2E cleanup left resources behind" && status=1- }+ if [ "$CLEANUP_DONE" -ne 1 ] && ! cleanup; then+ if [ "$status" -ne 0 ]; then+ warn 'Internal dependency E2E cleanup also failed; original failure is preserved'+ else+ fail 'Internal dependency E2E cleanup failed'+ status=1+ fifiexit "$status"}trap on_exit EXIT-if [ "${1:-}" = "--cleanup-only" ]; then- cleanup- exit $?-fi+[[ "$BACKEND_PORT" =~ ^[0-9]+$ ]] && [ "$BACKEND_PORT" -gt 0 ] &&+ [ "$BACKEND_PORT" -le 65535 ] || { fail 'Invalid E2E_BACKEND_PORT'; exit 1; }need_cmd() {command -v "$1" >/dev/null 2>&1 && ok "Command '$1' available" && return@@ -174,29 +164,31 @@ log "Context=$(current_e2e_context || true)"header "Phase 0 - Safety"need_cmd "$KUBECTL_BIN"require_safe_kube_context-kctl cluster-info >/dev/null 2>&1 && ok "Kubernetes cluster reachable" || {+if ! kctl cluster-info >/dev/null 2>&1; thenfail "Kubernetes cluster not reachable"exit 1-}+fi+ok "Kubernetes cluster reachable"for ns in "$WORKFLOW_RECIPE_NS" "$SANDBOX_NS" "$CONTROL_NS"; do- kctl get ns "$ns" >/dev/null 2>&1 && ok "Namespace ${ns} exists" || {+ if ! kctl get ns "$ns" >/dev/null 2>&1; thenfail "Namespace ${ns} not found"exit 1- }+ fi+ ok "Namespace ${ns} exists"done-kctl get crd workflowrecipes.clerum.io >/dev/null 2>&1 && ok "WorkflowRecipe CRD installed" || {+if ! kctl get crd workflowrecipes.clerum.io >/dev/null 2>&1; thenfail "WorkflowRecipe CRD not installed"exit 1-}-kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/null 2>&1 \- && ok "workflow-recipes rolled out" || {+fi+ok "WorkflowRecipe CRD installed"+if ! kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/null 2>&1; thenfail "workflow-recipes deployment is not ready"exit 1- }+fi+ok "workflow-recipes rolled out"-header "Phase 1 - Apply isolated fixture"-cleanup >/dev/null 2>&1 || true-cat <<YAML | kctl apply -f -+header "Phase 1 - Create isolated fixture"+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: clerum.io/v1alpha1kind: WorkflowRecipemetadata:@@ -224,12 +216,11 @@ spec:args:- "mkdir -p /www && printf 'issue485-ok\n' > /www/index.html && exec httpd -f -p ${BACKEND_PORT} -h /www"YAML-CREATED=1-ok "WorkflowRecipe fixture applied"+ok "Run-owned WorkflowRecipe fixture created"-if kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- -o jsonpath='{.spec.workloads[*].egressBindings}' | grep -q .; then- fail "Fixture contains egressBindings; that would hide the behavior under test"+if ! kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o json \+ | jq -e 'all(.spec.workloads[]; ((.egressBindings // []) | length) == 0)' >/dev/null; then+ fail "Could not verify the live fixture has no egressBindings shortcut"exit 1fiok "Fixture has no egressBindings shortcut"@@ -243,22 +234,25 @@ BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")"fail "Backend workload instance was not assigned"exit 1}-wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {+if ! wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD"; thenfail "Source deployment not ready"exit 1-}-wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {+fi+ok "Source deployment ready"+if ! wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD"; thenfail "Backend deployment not ready"exit 1-}+fi+ok "Backend deployment ready"wait_internal_ready "$TIMEOUT_POD"resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"-[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {+if [ "$resolved_target" != "$expected_target" ]; thenfail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"exit 1-}+fi+ok "TARGET_URL resolved to ${resolved_target}"header "Phase 3 - wr-intdep policy shape"egress_selector="clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME},clerum.io/source-workload=${SOURCE_ID}"@@ -269,14 +263,9 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"header "Phase 4 - Positive packet flow"-# shellcheck disable=SC2016-positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \- sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"-printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \- ok "WRC source reached backend through inferred internal dependency" || {- fail "WRC source could not reach backend (output: ${positive_output})"- exit 1- }+wrc_assert_http_allowed 'WRC source reached backend through inferred internal dependency' \+ "$SANDBOX_NS" "deploy/${SOURCE_DEPLOYMENT}" \+ "${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local" "$BACKEND_PORT" issue485-okheader "Phase 5 - Live drift repair and steady no-churn"egress_ns="${egress_ref%%/*}"@@ -292,41 +281,25 @@ wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120wrc_wait_for_np_spec_hash "$egress_ns" "$egress_name" "$egress_hash" 120wrc_wait_for_np_spec_hash "$ingress_ns" "$ingress_name" "$ingress_hash" 120-# shellcheck disable=SC2016-repaired_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \- sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"-printf "%s" "$repaired_output" | grep -Fq "issue485-ok" && \- ok "Internal dependency traffic recovered after live policy repair" || {- fail "Internal dependency traffic did not recover after policy repair"- exit 1- }+wrc_assert_http_allowed 'Internal dependency traffic recovered after live policy repair' \+ "$SANDBOX_NS" "deploy/${SOURCE_DEPLOYMENT}" \+ "${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local" "$BACKEND_PORT" issue485-ok-egress_rv="$(wrc_np_resource_version "$egress_ns" "$egress_name")"-ingress_rv="$(wrc_np_resource_version "$ingress_ns" "$ingress_name")"+wrc_begin_np_observation+wrc_track_np "$egress_ns" "$egress_name" internal-dependency+wrc_track_np "$ingress_ns" "$ingress_name" internal-dependencywrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120-wrc_wait_for_np_noop_witness "$egress_ns" "$egress_name" internal-dependency apply 120-wrc_wait_for_np_noop_witness "$ingress_ns" "$ingress_name" internal-dependency apply 120-sleep "$STABILITY_SECONDS"-[ "$(wrc_np_resource_version "$egress_ns" "$egress_name")" = "$egress_rv" ] || {- fail "${egress_ref} churned after convergence"- exit 1-}-[ "$(wrc_np_resource_version "$ingress_ns" "$ingress_name")" = "$ingress_rv" ] || {- fail "${ingress_ref} churned after convergence"- exit 1-}-ok "Both internal-dependency policies stayed resourceVersion-stable"+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120header "Phase 6 - Negative packet flow"-kctl delete pod "$DENIED_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true-cat <<YAML | kctl apply -f - >/dev/null+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: v1kind: Podmetadata:name: ${DENIED_POD}namespace: ${SANDBOX_NS}labels:- run: ${DENIED_POD}+ e2e.clerum.io/probe: ${DENIED_POD}e2e.clerum.io/suite: wrc-internal-dependency-networkpolicyspec:restartPolicy: Never@@ -345,24 +318,40 @@ spec:capabilities:drop: ["ALL"]YAML-wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod ready" || {- fail "Unlabeled negative pod not ready"- exit 1-}-if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \- --timeout="$CONNECT_TIMEOUT" --tries=1 \- "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then- fail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"+if ! wait_for_pod "$SANDBOX_NS" "e2e.clerum.io/probe=${DENIED_POD}" 60; then+ fail "Rogue probe pod not ready"exit 1fi-ok "Unlabeled pod cannot reach backend"+ok "Rogue probe pod ready"++# The rogue source must have a working egress path before testing backend+# ingress. Use the Service IP so denied DNS cannot masquerade as ingress denial.+backend_ip="$(kctl get service "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"+[[ "$backend_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || {+ fail 'Backend Service has no IPv4 ClusterIP for the isolated ingress control'+ exit 1+}+backend_pod_selector="$(kctl get deployment "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" -o json \+ | jq -ce '.spec.selector.matchLabels | select(type == "object" and length > 0)')"+probe_pod_selector="$(jq -cn --arg probe "$DENIED_POD" '{"e2e.clerum.io/probe":$probe}')"+wrc_create_connection_policy "$PROBE_EGRESS_POLICY" Egress \+ "$SANDBOX_NS" "$probe_pod_selector" "$SANDBOX_NS" "$backend_pod_selector" "$BACKEND_PORT"+wrc_create_connection_policy "$PROBE_INGRESS_POLICY" Ingress \+ "$SANDBOX_NS" "$backend_pod_selector" "$SANDBOX_NS" "$probe_pod_selector" "$BACKEND_PORT"+wrc_assert_http_allowed 'Same rogue source reaches healthy backend with temporary ingress' \+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT" issue485-ok+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$PROBE_INGRESS_POLICY"+wrc_assert_http_blocked 'Internal dependency ingress rejects a source outside its workload selector' \+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT"+wrc_create_connection_policy "$PROBE_INGRESS_POLICY" Ingress \+ "$SANDBOX_NS" "$backend_pod_selector" "$SANDBOX_NS" "$probe_pod_selector" "$BACKEND_PORT"+wrc_assert_http_allowed 'Same rogue source and backend remain healthy after the negative control' \+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT" issue485-ok+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$PROBE_INGRESS_POLICY"header "Phase 7 - Cleanup"cleanup-CREATED=0-kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" >/dev/null 2>&1 \- && fail "WorkflowRecipe still exists after cleanup" || ok "WorkflowRecipe removed"-policy_refs "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \- | grep -q . && fail "wr-intdep policies still exist after cleanup" || ok "wr-intdep policies removed"+CLEANUP_DONE=1+ok 'Run-owned fixture resources removed with UID-checked cleanup'print_resultsdiff --git a/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh b/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.shindex 73e8248bf..2224d3201 100755--- a/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh+++ b/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh@@ -6,6 +6,9 @@ set -euo pipefailSCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)": "${KUBECONTEXT:?KUBECONTEXT is required for WRC NetworkPolicy E2E}"+# shellcheck source=scripts/e2e/_lib/wrc-networkpolicy-convergence.sh+source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"+wrc_require_networkpolicy_leasesuites=(e2e-wrc-networkpolicy-service-routes.shdiff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.shindex 825d36f53..99d8afdc5 100755--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh@@ -8,61 +8,68 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${SCRIPT_DIR}/e2e-lib.sh"# shellcheck disable=SC1091source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"+# shellcheck disable=SC1091+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"+wrc_fixture_init-raw_run_id="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"-RUN_ID="$(printf '%s' "$raw_run_id" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g' | cut -c1-16)"-[ -n "$RUN_ID" ] || RUN_ID="run-$$"--RECIPE_NAME="${E2E_RECIPE_NAME:-e2e-wrc-routes-${RUN_ID}}"+RECIPE_NAME="e2e-routes-${E2E_RUN_ID}"SANDBOX_UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"UI_ID="ui"API_ID="api"DB_ID="db"+OTHER_DB_ID="other-db"DB_PORT=8081-CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"ROGUE_UI_POD="${RECIPE_NAME}-rogue-ui"ROGUE_WORKLOAD_POD="${RECIPE_NAME}-rogue-wl"CREATED=0HELD_POLICY_FINALIZER=0+HELD_POLICY_UID=""UI_DEPLOYMENT=""API_DEPLOYMENT=""DB_DEPLOYMENT=""+OTHER_DB_DEPLOYMENT=""UI_INGRESS_POLICY="ui-ingress-${RECIPE_NAME}-${API_ID}"WL_EGRESS_POLICY="wl-egress-${RECIPE_NAME}-${API_ID}"WL_INGRESS_POLICY="wl-ingress-${RECIPE_NAME}-${DB_ID}"cleanup() {- local status=0 namespace- kctl delete pod "$ROGUE_UI_POD" -n "$SANDBOX_UI_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1- kctl delete pod "$ROGUE_WORKLOAD_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1- if [ "$HELD_POLICY_FINALIZER" = "1" ] &&- kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" >/dev/null 2>&1; then- kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \- -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || status=1- HELD_POLICY_FINALIZER=0- fi- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" >/dev/null 2>&1 || status=1- for namespace in "$SANDBOX_NS" "$SANDBOX_UI_NS"; do- kctl delete networkpolicy -n "$namespace" -l "clerum.io/recipe=${RECIPE_NAME}" --ignore-not-found >/dev/null 2>&1 || status=1- done+ local status=0+ if [ "$HELD_POLICY_FINALIZER" = "1" ]; then release_policy_hold || status=1; fi+ wrc_cleanup_owned || status=1return "$status"}+release_policy_hold() {+ local policy patch+ policy="$(kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --ignore-not-found -o json)" || return 1+ [ -n "$policy" ] || { HELD_POLICY_FINALIZER=0; return 0; }+ patch="$(printf '%s' "$policy" | jq -ce --arg uid "$HELD_POLICY_UID" '+ select(.metadata.uid == $uid)+ | [{op:"test",path:"/metadata/uid",value:$uid},+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},+ {op:"replace",path:"/metadata/finalizers",value:[.metadata.finalizers[] | select(. != "e2e.invalid/hold-deletion")]}]')" || return 1+ kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=json -p "$patch" >/dev/null || return 1+ HELD_POLICY_FINALIZER=0+}+on_exit() {- local status=$?+ local status=$? cleanup_status=0+ trap - EXIT INT TERMif [ "$CREATED" = "1" ] && [ "${E2E_KEEP_RESOURCES:-0}" != "1" ]; then- cleanup >/dev/null 2>&1 || {- [ "$status" -ne 0 ] && warn "WRC service-route E2E cleanup left resources behind"- [ "$status" -eq 0 ] && fail "WRC service-route E2E cleanup left resources behind" && status=1- }+ cleanup || cleanup_status=$?+ fi+ if [ "$cleanup_status" -ne 0 ]; then+ warn 'WRC service-route E2E cleanup did not complete'+ if [ "$status" -eq 0 ]; then status=1; fifiexit "$status"}trap on_exit EXIT+trap 'exit 130' INT+trap 'exit 143' TERMwait_for_workload_instance() {local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance@@ -111,31 +118,9 @@ wait_for_recipe_message() {return 1}-assert_http_body() {- local description=$1 namespace=$2 deployment=$3 host=$4 port=$5 expected=$6 body- # shellcheck disable=SC2016- body="$(kctl exec "deploy/${deployment}" -n "$namespace" -- sh -c 'printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$1" "$2" "$3"' -- "$CONNECT_TIMEOUT" "$host" "$port" 2>/dev/null || true)"- if printf '%s' "$body" | grep -Fq "$expected"; then- ok "$description"- return 0- fi- fail "${description} (unexpected response)"- return 1-}--assert_pod_blocked() {- local description=$1 namespace=$2 pod=$3 host=$4 port=$5- # shellcheck disable=SC2016- if kctl exec "$pod" -n "$namespace" -- sh -c 'printf "GET / HTTP/1.0\r\n\r\n" | nc -w "$1" "$2" "$3"' -- "$CONNECT_TIMEOUT" "$host" "$port" >/dev/null 2>&1; then- fail "${description}: unexpected connection succeeded"- return 1- fi- ok "$description"-}-apply_rogue_pod() {local namespace=$1 name=$2- cat <<YAML | kctl apply -f - >/dev/null+ cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: v1kind: Podmetadata:@@ -164,26 +149,6 @@ YAMLwait_for_pod "$namespace" "e2e.clerum.io/probe=${name}" 60}-assert_all_stable() {- local ui_before wl_egress_before wl_ingress_before- local ui_after wl_egress_after wl_ingress_after- ui_before="$(wrc_np_resource_version "$SANDBOX_NS" "$UI_INGRESS_POLICY")"- wl_egress_before="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_EGRESS_POLICY")"- wl_ingress_before="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_INGRESS_POLICY")"-- # Intentional no-churn observation window after all readiness assertions.- sleep "$STABILITY_SECONDS"-- ui_after="$(wrc_np_resource_version "$SANDBOX_NS" "$UI_INGRESS_POLICY")"- wl_egress_after="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_EGRESS_POLICY")"- wl_ingress_after="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_INGRESS_POLICY")"-- [ "$ui_after" = "$ui_before" ] || fail "${UI_INGRESS_POLICY} churned"- [ "$wl_egress_after" = "$wl_egress_before" ] || fail "${WL_EGRESS_POLICY} churned"- [ "$wl_ingress_after" = "$wl_ingress_before" ] || fail "${WL_INGRESS_POLICY} churned"- ok "All three converged NetworkPolicies stayed resourceVersion-stable"-}-header "WRC NetworkPolicy correctly-configured service routes"require_safe_kube_contextkctl cluster-info >/dev/null 2>&1 || {@@ -193,8 +158,8 @@ kctl cluster-info >/dev/null 2>&1 || {kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/nullheader "Phase 1 — isolated fixture"-cleanup >/dev/null 2>&1 || true-cat <<YAML | kctl apply -f - >/dev/null+CREATED=1+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_ownedapiVersion: clerum.io/v1alpha1kind: WorkflowRecipemetadata:@@ -230,6 +195,13 @@ spec:command: ["sh", "-c"]args:- "mkdir -p /www && printf 'db-ok\n' > /www/index.html && exec httpd -f -p ${DB_PORT} -h /www"+ - id: ${OTHER_DB_ID}+ type: deployment+ image: busybox:1.36.1+ port: ${DB_PORT}+ command: ["sh", "-c"]+ args:+ - "mkdir -p /www && printf 'other-db-ok\n' > /www/index.html && exec httpd -f -p ${DB_PORT} -h /www"ui:workloadRef: ${UI_ID}port: 8080@@ -240,12 +212,12 @@ spec:- workloadRef: ${API_ID}port: 8080YAML-CREATED=1wait_for_recipe_active 180UI_DEPLOYMENT="$(wait_for_workload_instance "$UI_ID" 120)"API_DEPLOYMENT="$(wait_for_workload_instance "$API_ID" 120)"DB_DEPLOYMENT="$(wait_for_workload_instance "$DB_ID" 120)"+OTHER_DB_DEPLOYMENT="$(wait_for_workload_instance "$OTHER_DB_ID" 120)"[ -n "$UI_DEPLOYMENT" ] && [ -n "$API_DEPLOYMENT" ] && [ -n "$DB_DEPLOYMENT" ] || {fail "One or more workload instance names are missing"exit 1@@ -254,6 +226,7 @@ DB_DEPLOYMENT="$(wait_for_workload_instance "$DB_ID" 120)"wait_for_deployment "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" 180wait_for_deployment "$SANDBOX_NS" "$API_DEPLOYMENT" 180wait_for_deployment "$SANDBOX_NS" "$DB_DEPLOYMENT" 180+wait_for_deployment "$SANDBOX_NS" "$OTHER_DB_DEPLOYMENT" 180for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; dowrc_wait_for_np "$SANDBOX_NS" "$policy" 120@@ -262,20 +235,58 @@ ok "ui-ingress, workload-egress and workload-ingress materialized"API_CLUSTER_IP="$(kctl get service "$API_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"DB_CLUSTER_IP="$(kctl get service "$DB_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"+OTHER_DB_CLUSTER_IP="$(kctl get service "$OTHER_DB_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"[ -n "$API_CLUSTER_IP" ] && [ -n "$DB_CLUSTER_IP" ] || {fail "Fixture Service ClusterIP discovery failed"exit 1}header "Phase 2 — correctly configured positive routes"-assert_http_body "sandbox UI reaches declared API backend" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"-assert_http_body "API workload reaches declared sibling DB" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"+wrc_assert_http_allowed "sandbox UI reaches declared API backend" "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"+wrc_assert_http_allowed "API workload reaches declared sibling DB" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"header "Phase 3 — negative controls"apply_rogue_pod "$SANDBOX_UI_NS" "$ROGUE_UI_POD"apply_rogue_pod "$SANDBOX_NS" "$ROGUE_WORKLOAD_POD"-assert_pod_blocked "unlabelled sandbox-ui pod cannot reach recipe API" "$SANDBOX_UI_NS" "$ROGUE_UI_POD" "$API_CLUSTER_IP" 8080-assert_pod_blocked "unlabelled sandbox workload cannot reach recipe DB" "$SANDBOX_NS" "$ROGUE_WORKLOAD_POD" "$DB_CLUSTER_IP" "$DB_PORT"+api_selector="$(kctl get deployment "$API_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"+db_selector="$(kctl get deployment "$DB_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"+other_db_selector="$(kctl get deployment "$OTHER_DB_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"+ui_probe_selector="$(jq -cn --arg probe "$ROGUE_UI_POD" '{"e2e.clerum.io/probe":$probe}')"+wl_probe_selector="$(jq -cn --arg probe "$ROGUE_WORKLOAD_POD" '{"e2e.clerum.io/probe":$probe}')"++# Leave only the ingress under test as the rejecting boundary. The same rogue+# must first prove it can execute and reach the healthy backend with a narrow+# temporary ingress permission, and must lose access when that permission goes.+wrc_create_connection_policy "${RECIPE_NAME}-ui-source" Egress "$SANDBOX_UI_NS" "$ui_probe_selector" "$SANDBOX_NS" "$api_selector" 8080+wrc_create_connection_policy "${RECIPE_NAME}-wl-source" Egress "$SANDBOX_NS" "$wl_probe_selector" "$SANDBOX_NS" "$db_selector" "$DB_PORT"+for lane in ui wl; do+ if [ "$lane" = ui ]; then+ probe_ns=$SANDBOX_UI_NS; probe=$ROGUE_UI_POD; probe_selector=$ui_probe_selector+ target_selector=$api_selector; target_ip=$API_CLUSTER_IP; target_port=8080; expected=api-ok+ else+ probe_ns=$SANDBOX_NS; probe=$ROGUE_WORKLOAD_POD; probe_selector=$wl_probe_selector+ target_selector=$db_selector; target_ip=$DB_CLUSTER_IP; target_port=$DB_PORT; expected=db-ok+ fi+ control="${RECIPE_NAME}-${lane}-target"+ wrc_create_connection_policy "$control" Ingress "$SANDBOX_NS" "$target_selector" "$probe_ns" "$probe_selector" "$target_port"+ wrc_assert_http_allowed "$lane negative probe reaches the healthy target with its control permission" "$probe_ns" "$probe" "$target_ip" "$target_port" "$expected"+ wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$control"+ wrc_assert_http_blocked "$lane ingress rejects the rogue when only its target permission is removed" "$probe_ns" "$probe" "$target_ip" "$target_port"+ wrc_create_connection_policy "$control" Ingress "$SANDBOX_NS" "$target_selector" "$probe_ns" "$probe_selector" "$target_port"+ wrc_assert_http_allowed "$lane negative control remains healthy after the denial" "$probe_ns" "$probe" "$target_ip" "$target_port" "$expected"+ wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$control"+done++# For workload egress, allow only the complementary destination ingress. An+# undeclared sibling must remain unreachable from the actual API workload.+wrc_create_connection_policy "${RECIPE_NAME}-other-target" Ingress "$SANDBOX_NS" "$other_db_selector" "$SANDBOX_NS" "$api_selector" "$DB_PORT"+wrc_create_connection_policy "${RECIPE_NAME}-other-source" Egress "$SANDBOX_NS" "$api_selector" "$SANDBOX_NS" "$other_db_selector" "$DB_PORT"+wrc_assert_http_allowed 'API probe reaches undeclared sibling with temporary egress permission' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT" 'other-db-ok'+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "${RECIPE_NAME}-other-source"+wrc_assert_http_blocked 'Workload egress rejects the undeclared sibling with destination ingress allowed' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT"+wrc_create_connection_policy "${RECIPE_NAME}-other-source" Egress "$SANDBOX_NS" "$api_selector" "$SANDBOX_NS" "$other_db_selector" "$DB_PORT"+wrc_assert_http_allowed 'Undeclared sibling remains healthy after egress denial' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT" 'other-db-ok'+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "${RECIPE_NAME}-other-source"header "Phase 4 — live drift repair"ui_hash="$(wrc_np_spec_hash "$SANDBOX_NS" "$UI_INGRESS_POLICY")"@@ -292,41 +303,47 @@ wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_EGRESS_POLICY" "$wl_egress_hash" 12wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$wl_ingress_hash" 120ok "WRC repaired live spec drift for all three families"-assert_http_body "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"-assert_http_body "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"+wrc_assert_http_allowed "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"+wrc_assert_http_allowed "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"header "Phase 5 — terminating race self-heals without another parent event"terminating_uid="$(wrc_np_uid "$SANDBOX_NS" "$WL_INGRESS_POLICY")"+HELD_POLICY_UID=$terminating_uidterminating_generation="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')"-kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \- -p '{"metadata":{"finalizers":["e2e.invalid/hold-deletion"]}}' >/dev/null+held_policy="$(kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" -o json)"+printf '%s' "$held_policy" | wrc_record_owned+hold_patch="$(printf '%s' "$held_policy" | jq -c '[+ {op:"test",path:"/metadata/uid",value:.metadata.uid},+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},+ {op:"add",path:"/metadata/finalizers",value:((.metadata.finalizers // []) + ["e2e.invalid/hold-deletion"])}]')"+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=json -p "$hold_patch" >/dev/nullHELD_POLICY_FINALIZER=1-kctl delete networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --wait=false >/dev/null+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$WL_INGRESS_POLICY" falsewrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120wait_for_recipe_message 'is terminating; retrying after deletion' 120-kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \- -p '{"metadata":{"finalizers":[]}}' >/dev/null-HELD_POLICY_FINALIZER=0+release_policy_holdwrc_wait_for_np_recreated "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$terminating_uid" "$wl_ingress_hash" 120[ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')" = "$((terminating_generation + 1))" ] || {fail "NetworkPolicy recovery depended on an unexpected second parent spec event"exit 1}wait_for_recipe_active 120-assert_http_body "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"+wrc_assert_http_allowed "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"header "Phase 6 — steady-state no-churn"+wrc_begin_np_observation+wrc_track_np "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress+wrc_track_np "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter+wrc_track_np "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingresswrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress apply 120-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter 120-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingress apply 120-assert_all_stable+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120header "Phase 7 — cleanup"cleanupCREATED=0for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; do- if kctl get networkpolicy "$policy" -n "$SANDBOX_NS" >/dev/null 2>&1; then+ remaining="$(kctl get networkpolicy "$policy" -n "$SANDBOX_NS" --ignore-not-found -o name)"+ if [ -n "$remaining" ]; thenfail "NetworkPolicy ${SANDBOX_NS}/${policy} survived cleanup"exit 1fidiff --git a/scripts/tests/test-wrc-fixtures.sh b/scripts/tests/test-wrc-fixtures.shnew file mode 100755index 000000000..7fd15cded--- /dev/null+++ b/scripts/tests/test-wrc-fixtures.sh@@ -0,0 +1,227 @@+#!/usr/bin/env bash+# Hermetic API/process boundary contracts; never invokes kubectl or a cluster.+set -euo pipefail+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+# shellcheck disable=SC1091+source "$ROOT/scripts/e2e/_lib/wrc-fixtures.sh"+TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-fixtures.XXXXXX")"+trap 'rm -rf -- "$TEST_ROOT"' EXIT+WRC_FIXTURE_DIR="$TEST_ROOT"+WRC_FIXTURE_LEDGER="$TEST_ROOT/owned.jsonl"+E2E_RUN_ID=contract123+MODE=normal++reset_fixture() {+ MODE=normal+ : > "$WRC_FIXTURE_LEDGER"+ : > "$TEST_ROOT/calls"+ rm -f "$TEST_ROOT/live.json"+}++kctl() {+ printf '%s\n' "$*" >> "$TEST_ROOT/calls"+ case "$1" in+ create)+ [[ ! -e "$TEST_ROOT/live.json" ]] || return 1+ jq '.metadata.uid="created-uid"' > "$TEST_ROOT/live.json"+ cat "$TEST_ROOT/live.json"+ ;;+ get)+ [[ "$MODE" != read-error ]] || return 1+ if [[ "$2" == deployment,service,configmap,networkpolicy ]]; then+ cat "$TEST_ROOT/children.json"+ return+ fi+ [[ -e "$TEST_ROOT/live.json" ]] || return 0+ if [[ " $* " == *' -o name '* ]]; then+ printf 'configmap/fixture\n'+ else+ cat "$TEST_ROOT/live.json"+ fi+ ;;+ delete)+ [[ "$2" == --raw && "$3" == /api/v1/namespaces/sandbox-recipes/configmaps/fixture ]] || return 1+ [[ "$4" == --request-timeout=30s && "$5" == -f ]] || return 1+ jq -e '.kind=="DeleteOptions" and .preconditions.uid=="created-uid"' "$6" >/dev/null || return 1+ [[ "$MODE" != delete-error ]] || return 1+ if [[ "$MODE" == recreate-race ]]; then+ jq '.metadata.uid="replacement-uid"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"+ mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"+ fi+ # Simulate the API server's atomic precondition rather than allowing a+ # name-only delete to make the ownership-race contract vacuously green.+ [[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == "$(jq -r .preconditions.uid "$6")" ]] || return 1+ rm "$TEST_ROOT/live.json"+ ;;+ wait) [[ ! -e "$TEST_ROOT/live.json" ]] ;;+ *) return 1 ;;+ esac+}++create_fixture() {+ jq -n '{apiVersion:"v1",kind:"ConfigMap",metadata:{namespace:"sandbox-recipes",name:"fixture"},+ data:{message:"not retained in ownership evidence"}}' | wrc_create_owned+}++must_reject() {+ if "$@" > "$TEST_ROOT/rejected.out" 2>&1; then+ echo "Expected rejection: $*" >&2+ exit 1+ fi+}++reset_fixture+create_fixture+jq -e '.metadata.uid=="created-uid" and .metadata.labels["e2e.clerum.io/run"]=="contract123" and (has("data")|not)' \+ "$WRC_FIXTURE_LEDGER" >/dev/null+must_reject create_fixture+[[ "$(wc -l < "$WRC_FIXTURE_LEDGER" | tr -d ' ')" == 1 ]]+[[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == created-uid ]]+echo 'PASS: collision preserves prior object and does not enroll it twice'++MODE=read-error+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture+[[ "$(rg -c '^delete ' "$TEST_ROOT/calls" || true)" == '' ]]+MODE=normal+jq '.metadata.uid="replacement-uid"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture+[[ "$(rg -c '^delete ' "$TEST_ROOT/calls" || true)" == '' ]]+echo 'PASS: read errors and changed UID refuse deletion'++reset_fixture+create_fixture+jq '.metadata.labels["e2e.clerum.io/run"]="foreign"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture+[[ -e "$TEST_ROOT/live.json" ]]+echo 'PASS: changed ownership label refuses deletion'++reset_fixture+create_fixture+jq '.metadata.labels["controller-added"]="retained"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"+wrc_delete_owned sandbox-recipes ConfigMap fixture+[[ ! -e "$TEST_ROOT/live.json" ]]+echo 'PASS: extra controller labels accepted; DELETE carries exact UID and proves absence'++reset_fixture+create_fixture+MODE=recreate-race+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture+[[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == replacement-uid ]]+echo 'PASS: API UID precondition preserves concurrent replacement'++reset_fixture+create_fixture+MODE=delete-error+must_reject wrc_cleanup_owned+[[ -e "$TEST_ROOT/live.json" ]]+echo 'PASS: cleanup failure is nonzero and leaves evidence'++reset_fixture+# Only owner UID, complete cross-namespace recipe binding, or WRC's exact+# same-namespace managed recipe label can enroll a generated child.+jq -n '{items:[+ {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"ownerless",namespace:"sandbox-recipes",uid:"child1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"fixture"}}},+ {apiVersion:"v1",kind:"Service",metadata:{name:"cross-ns",namespace:"sandbox-ui",uid:"child2",labels:{"clerum.io/recipe-namespace":"sandbox-recipes","clerum.io/recipe-name":"fixture"}}},+ {apiVersion:"apps/v1",kind:"Deployment",metadata:{name:"owned",namespace:"sandbox-recipes",uid:"child3",ownerReferences:[{uid:"recipe-uid"}]}},+ {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"foreign",namespace:"sandbox-recipes",uid:"foreign1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"another"}}},+ {apiVersion:"v1",kind:"Service",metadata:{name:"foreign-ns",namespace:"sandbox-ui",uid:"foreign2",labels:{"clerum.io/recipe-namespace":"other-namespace","clerum.io/recipe-name":"fixture"}}}+]}' > "$TEST_ROOT/children.json"+wrc_capture_recipe_children sandbox-recipes fixture recipe-uid+jq -se 'map(.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/null+echo 'PASS: generated ownerless/cross-namespace children captured without foreign resources'++# Validate rejected IDs before any fixture directory or cluster operation.+wrc_require_networkpolicy_lease() { return 0; }+for invalid in 'UPPER' 'contains-dash' '../escape' '1234567890123'; do+ E2E_RUN_ID="$invalid" must_reject wrc_fixture_init+done+echo 'PASS: invalid run IDs fail without lossy normalization'++printf 'Forwarding from 127.0.0.1:38123 -> 8080\n' > "$TEST_ROOT/port-forward.log"+[[ "$(wrc_port_forward_reported_port 8080)" == 38123 ]]+must_reject wrc_port_forward_reported_port 8090+printf 'Forwarding from [::1]:38123 -> 8080\n' > "$TEST_ROOT/port-forward.log"+must_reject wrc_port_forward_reported_port 8080+printf 'Forwarding from 127.0.0.1:70000 -> 8080\n' > "$TEST_ROOT/port-forward.log"+must_reject wrc_port_forward_reported_port 8080+printf 'Forwarding from 127.0.0.1:38123 -> 8080\nForwarding from 127.0.0.1:39123 -> 8080\n' > "$TEST_ROOT/port-forward.log"+must_reject wrc_port_forward_reported_port 8080+echo 'PASS: dynamic binding comes only from one valid matching IPv4 child report'++# These process doubles exercise fail-closed cleanup without launching a+# process, touching user forwards, or relying on machine-specific ps output.+WRC_PORT_FORWARD_PID=99999+WRC_PORT_FORWARD_START=original+WRC_PORT_FORWARD_COMMAND='kubectl --context=owned -n sandbox-recipes port-forward --address=127.0.0.1 svc/fixture :8080'+PROCESS_STATE=live+PROCESS_START=original+PROCESS_COMMAND="$WRC_PORT_FORWARD_COMMAND"+SIGNALLED=0+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" "$WRC_PORT_FORWARD_PID" \+ "$WRC_PORT_FORWARD_START" owned owned "$ROOT" sandbox-recipes fixture 38123 8080+pf_owner_process_state() { printf '%s\n' "$PROCESS_STATE"; }+pf_owner_process_start() { printf '%s\n' "$PROCESS_START"; }+pf_owner_process_command() { printf '%s\n' "$PROCESS_COMMAND"; }+pf_owner_signal_process() { SIGNALLED=$((SIGNALLED + 1)); PROCESS_STATE=dead; }+pf_owner_reap_process() { :; }+wrc_assert_port_forward+PROCESS_START=reused+must_reject wrc_stop_port_forward+[[ "$SIGNALLED" == 0 ]]+PROCESS_START=original+PROCESS_COMMAND='foreign process'+must_reject wrc_stop_port_forward+[[ "$SIGNALLED" == 0 ]]+PROCESS_COMMAND="$WRC_PORT_FORWARD_COMMAND"+PROCESS_STATE=dead+must_reject wrc_stop_port_forward+[[ "$SIGNALLED" == 0 ]]+PROCESS_STATE=live+wrc_stop_port_forward+[[ "$SIGNALLED" == 1 && -z "$WRC_PORT_FORWARD_PID" ]]+[[ ! -e "$TEST_ROOT/port-forward.pid" ]] || {+ echo 'FAIL: normal exit left the dead owned port-forward record' >&2+ exit 1+}+echo 'PASS: dead/reused/foreign forwards fail; exact live child is stopped and reaped'++WRC_PORT_FORWARD_PID=99999+SIGNALLED=0+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" "$WRC_PORT_FORWARD_PID" \+ "$WRC_PORT_FORWARD_START" owned owned "$ROOT" sandbox-recipes fixture 38123 8080+printf '0\n' > "$TEST_ROOT/process-reads"+pf_owner_process_state() {+ if [[ "$SIGNALLED" == 0 ]]; then+ printf 'live\n'+ return+ fi+ local reads+ reads="$(cat "$TEST_ROOT/process-reads")"+ printf '%s\n' "$((reads + 1))" > "$TEST_ROOT/process-reads"+ if [[ "$reads" -lt 2 ]]; then printf 'live\n'; else printf 'dead\n'; fi+}+pf_owner_process_start() {+ # Model death between the post-TERM state check and the identity read.+ [[ "$SIGNALLED" == 0 ]] || return 1+ printf '%s\n' "$PROCESS_START"+}+wrc_stop_port_forward+[[ "$SIGNALLED" == 1 && -z "$WRC_PORT_FORWARD_PID" && ! -e "$TEST_ROOT/port-forward.pid" ]]+echo 'PASS: death during post-TERM identity read also removes the owned record'++WRC_PORT_FORWARD_PID=99999+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" 99998 original \+ owned owned "$ROOT" sandbox-recipes fixture 38123 8080+must_reject wrc_reap_port_forward+[[ -e "$TEST_ROOT/port-forward.pid" && "$WRC_PORT_FORWARD_PID" == 99999 ]]+rm "$TEST_ROOT/port-forward.pid"+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" 99999 foreign-start \+ owned owned "$ROOT" sandbox-recipes fixture 38123 8080+must_reject wrc_reap_port_forward+[[ -e "$TEST_ROOT/port-forward.pid" && "$WRC_PORT_FORWARD_PID" == 99999 ]]+echo 'PASS: cleanup preserves records with a different PID or process start'++echo 'WRC_FIXTURES_CONTRACT_PASS'diff --git a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.shindex 7a6c11298..087d02ec9 100755--- a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh+++ b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh@@ -1,19 +1,27 @@#!/usr/bin/env bash-set -u+# Hermetic orchestration contract. These receipts prove aggregate execution and+# failure propagation, not NetworkPolicy convergence or real dataplane traffic.+set -euo pipefailROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"AGGREGATE="${ROOT}/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh"-ROUTES="${ROOT}/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh"-INTDEP="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"-OAUTH="${ROOT}/scripts/e2e/e2e-sandbox-ui-oauth.sh"-WEBHOOK="${ROOT}/scripts/e2e/e2e-webhooks-basic.sh"-HELPER="${ROOT}/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"+FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-np-aggregate-contract.XXXXXX")"+trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT++suites=(+ e2e-wrc-networkpolicy-service-routes.sh+ e2e-wrc-internal-dependency-networkpolicy.sh+ e2e-sandbox-ui-oauth.sh+ e2e-webhooks-basic.sh+)+CONTEXT='owned-profile-contract'FAIL=0pass() { printf 'PASS: %s\n' "$1"; }-fail() { printf 'FAIL: %s\n' "$1"; FAIL=1; }+fail() { printf 'FAIL: %s\n' "$1" >&2; FAIL=1; }-for script in "$AGGREGATE" "$ROUTES" "$INTDEP" "$OAUTH" "$WEBHOOK" "$HELPER"; do+for script in "$AGGREGATE" "${suites[@]/#/${ROOT}/scripts/e2e/}" \+ "${ROOT}/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"; doif bash -n "$script"; thenpass "$(basename "$script") has valid bash syntax"else@@ -21,128 +29,132 @@ for script in "$AGGREGATE" "$ROUTES" "$INTDEP" "$OAUTH" "$WEBHOOK" "$HELPER"; dofidone-for suite in \- e2e-wrc-networkpolicy-service-routes.sh \- e2e-wrc-internal-dependency-networkpolicy.sh \- e2e-sandbox-ui-oauth.sh \- e2e-webhooks-basic.sh; do- grep -Fq "$suite" "$AGGREGATE" || fail "aggregate omits $suite"-done-if grep -Fq 'WRC_NETWORKPOLICY_E2E_PASS' "$AGGREGATE" &&- grep -Fq 'set -euo pipefail' "$AGGREGATE"; then- pass "aggregate is fail-loud and emits a terminal suite count"-else- fail "aggregate can pass without executing every suite"-fi-+# This is a static wiring check only. Behaviour is exercised below.if grep -Fq 'e2e-wrc-networkpolicy-live-convergence.sh' "${ROOT}/Makefile"; then- pass "canonical Make E2E aggregate includes the WRC NetworkPolicy gate"+ pass 'Makefile references the aggregate entry point (static wiring)'else- fail "canonical Make E2E aggregate omits the WRC NetworkPolicy gate"+ fail 'Makefile does not reference the aggregate entry point'fi-if grep -Fq 'patch workflowrecipe' "$HELPER" &&- grep -Fq 'metadata.generation' "$HELPER" &&- grep -Fq 'deployment/${controller_deployment}' "$HELPER"; then- pass "reconcile trigger advances the parent spec and observes the real WRC pass"-else- fail "reconcile trigger does not prove a parent recipe reconcile"-fi+mkdir -p "$FIXTURE_ROOT/scripts/e2e/_lib"+cp "$AGGREGATE" "$FIXTURE_ROOT/aggregate-original.sh"-if grep -Fq 'network policy unchanged; skipping update' "$HELPER" &&- grep -Fq 'network policy egress set unchanged; skipping live apply' "$HELPER" &&- grep -Fq '.policy == $policy' "$HELPER" &&- grep -Fq '.namespace == $namespace' "$HELPER" &&- grep -Fq '.family == $family' "$HELPER"; then- pass "no-churn witness binds structured WRC output to one policy and family"-else- fail "no-churn witness can pass without proving the named policy reached its no-op path"+# Only the fixture copy substitutes the external runtime/lease boundary. The+# production aggregate has no flag or environment variable to bypass it.+cat > "$FIXTURE_ROOT/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh" <<'STUB'+wrc_require_networkpolicy_lease() {+ : "${KUBECONTEXT:?KUBECONTEXT is required}"+ printf 'lease|%s\n' "$KUBECONTEXT" >> "$CONTRACT_RECEIPTS"+ return "${CONTRACT_LEASE_EXIT:-0}"+}+STUB++write_children() {+ local suite+ for suite in "${suites[@]}"; do+ cat > "$FIXTURE_ROOT/scripts/e2e/$suite" <<'CHILD'+#!/usr/bin/env bash+set -euo pipefail+suite="${0##*/}"+printf 'child|%s|%s\n' "$suite" "${KUBECONTEXT-<missing>}" >> "$CONTRACT_RECEIPTS"+if [ "$suite" = "${CONTRACT_FAIL_SUITE:-}" ]; then+ exit "$CONTRACT_CHILD_EXIT"fi+CHILD+ done+}-for entry in \- "$ROUTES:service-route" \- "$INTDEP:internal-dependency" \- "$OAUTH:OAuth broker" \- "$WEBHOOK:webhook"; do- script=${entry%%:*}- label=${entry#*:}- trigger_count=$(grep -Fc 'wrc_trigger_recipe_reconcile' "$script" || true)- if [ "$trigger_count" -ge 2 ]; then- pass "${label} E2E drives both repair and equivalent no-op reconciles"- else- fail "${label} E2E does not drive both required reconciles"+# Verify the exact receipts, not a name or counter appearing in source/output.+# Prefix length also proves no child after a failed/missing child was executed.+check_run() {+ local variant="$1" label="$2" expected_status="$3" child_count="$4" context_mode="$5"+ local failed_suite="${6:-}" child_exit="${7:-0}" lease_exit="${8:-0}" status=0 index marker_count+ : > "$FIXTURE_ROOT/receipts"+ : > "$FIXTURE_ROOT/expected"+ if [ "$context_mode" = 'present' ]; then+ printf 'lease|%s\n' "$CONTEXT" >> "$FIXTURE_ROOT/expected"+ for ((index = 0; index < child_count; index++)); do+ printf 'child|%s|%s\n' "${suites[$index]}" "$CONTEXT" >> "$FIXTURE_ROOT/expected"+ donefi-done-for entry in \- "$ROUTES:3:service-route" \- "$INTDEP:2:internal-dependency" \- "$OAUTH:1:OAuth broker" \- "$WEBHOOK:3:webhook"; do- script=${entry%%:*}- remainder=${entry#*:}- minimum=${remainder%%:*}- label=${remainder#*:}- witness_count=$(grep -Fc 'wrc_wait_for_np_noop_witness' "$script" || true)- if [ "$witness_count" -ge "$minimum" ]; then- pass "${label} E2E requires a no-op witness for every in-scope policy"- else- fail "${label} E2E has ${witness_count}/${minimum} required no-op witnesses"- fi-done+ local environment=(+ env -i PATH=/usr/bin:/bin+ "CONTRACT_RECEIPTS=$FIXTURE_ROOT/receipts"+ "CONTRACT_FAIL_SUITE=$failed_suite" "CONTRACT_CHILD_EXIT=$child_exit"+ "CONTRACT_LEASE_EXIT=$lease_exit"+ )+ case "$context_mode" in+ present) environment+=("KUBECONTEXT=$CONTEXT") ;;+ empty) environment+=(KUBECONTEXT=) ;;+ absent) ;;+ *) printf 'invalid contract context mode\n' >&2; return 1 ;;+ esac+ "${environment[@]}" bash "$FIXTURE_ROOT/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh" \+ > "$FIXTURE_ROOT/output" 2>&1 || status=$?-for entry in "$OAUTH:OAuth broker" "$WEBHOOK:webhook"; do- script=${entry%%:*}- label=${entry#*:}- if grep -Fq 'cleanup_status=1' "$script" &&- grep -Fq 'cleanup also failed' "$script" &&- grep -Fq 'status=1' "$script"; then- pass "${label} E2E makes cleanup failure fail the aggregate"- else- fail "${label} E2E cleanup can be silently ignored"+ marker_count="$(awk '/WRC_NETWORKPOLICY_E2E_PASS/ { count++ } END { print count+0 }' "$FIXTURE_ROOT/output")"+ if [ "$status" -ne "$expected_status" ] ||+ ! cmp -s "$FIXTURE_ROOT/expected" "$FIXTURE_ROOT/receipts" ||+ { [ "$expected_status" -eq 0 ] &&+ { [ "$marker_count" -ne 1 ] || [ "$(tail -n 1 "$FIXTURE_ROOT/output")" != 'WRC_NETWORKPOLICY_E2E_PASS suites=4' ]; }; } ||+ { [ "$expected_status" -ne 0 ] && [ "$marker_count" -ne 0 ]; }; then+ if [ "$variant" = original ]; then+ printf 'FAIL: %s (exit=%s expected=%s, PASS markers=%s)\n' "$label" "$status" "$expected_status" "$marker_count" >&2+ diff -u "$FIXTURE_ROOT/expected" "$FIXTURE_ROOT/receipts" >&2 || true+ fi+ return 1fi-done+ if [ "$variant" = original ]; then pass "$label"; fi+}-for family in UI_INGRESS_POLICY WL_EGRESS_POLICY WL_INGRESS_POLICY; do- grep -Fq "$family" "$ROUTES" || fail "service-route E2E omits $family"-done-if grep -Fq 'correctly configured positive routes' "$ROUTES" &&- grep -Fq 'negative controls' "$ROUTES" &&- grep -Fq 'live drift repair' "$ROUTES" &&- grep -Fq 'terminating race self-heals' "$ROUTES" &&- grep -Fq 'wrc_wait_for_np_recreated' "$ROUTES" &&- grep -Fq 'unexpected second parent spec event' "$ROUTES" &&- grep -Fq 'steady-state no-churn' "$ROUTES"; then- pass "UI/workload route E2E proves positive, negative, repair, scheduled retry, and no-churn signals"-else- fail "UI/workload route E2E is missing a required signal"-fi+verify_aggregate() {+ local variant="$1" rejected=0 index child_exit+ cp "$FIXTURE_ROOT/aggregate-$variant.sh" "$FIXTURE_ROOT/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh"+ write_children+ check_run "$variant" 'lease then four unique suites execute in order with the explicit context' 0 4 present || rejected=1+ for ((index = 0; index < ${#suites[@]}; index++)); do+ child_exit=$((41 + index))+ check_run "$variant" "child $((index + 1)) failure preserves exit $child_exit and prevents later children/PASS" \+ "$child_exit" "$((index + 1))" present "${suites[$index]}" "$child_exit" || rejected=1+ done+ for ((index = 0; index < ${#suites[@]}; index++)); do+ rm -- "$FIXTURE_ROOT/scripts/e2e/${suites[$index]}"+ check_run "$variant" "missing child $((index + 1)) fails before later children/PASS" 1 "$index" present || rejected=1+ write_children+ done+ check_run "$variant" 'absent context fails before the lease or any child' 1 0 absent || rejected=1+ check_run "$variant" 'empty context fails before the lease or any child' 1 0 empty || rejected=1+ check_run "$variant" 'lease rejection propagates before any child or PASS' 67 0 present '' 0 67 || rejected=1+ return "$rejected"+}-if grep -Fq 'Live drift repair and steady no-churn' "$INTDEP" &&- grep -Fq 'Internal dependency traffic recovered' "$INTDEP"; then- pass "internal-dependency E2E proves live repair and restored traffic"-else- fail "internal-dependency E2E lacks repair or traffic evidence"-fi+if ! verify_aggregate original; then fail 'real aggregate violates its execution contract'; fi-if grep -Fq 'BACKGROUND_POLICY_NAME' "$OAUTH" &&- grep -Fq 'Opted-in background workload reaches' "$OAUTH" &&- grep -Fq 'Unlabelled sandbox workload cannot use' "$OAUTH" &&- grep -Fq 'OAuth broker route recovered' "$OAUTH"; then- pass "OAuth broker E2E proves allow, deny, repair, and no-churn behavior"-else- fail "OAuth broker E2E lacks a required dataplane signal"-fi--for policy in PROXY_INGRESS_POLICY HANDLER_EGRESS_POLICY HANDLER_INGRESS_POLICY; do- grep -Fq "$policy" "$WEBHOOK" || fail "webhook E2E omits $policy"+# Mutation controls prove the contract cannot be fooled by keeping the suite+# names, executed counter, set -e, and terminal PASS text while removing the work+# or masking child failure. They mutate only a disposable aggregate copy.+for mutation in skipped-child ignored-error; do+ if ! awk -v mutation="$mutation" '+ /KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/ {+ count+++ if (mutation == "skipped-child") {+ sub(/KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/, ": # skipped child")+ } else {+ sub(/KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/, "KUBECONTEXT=\"$KUBECONTEXT\" bash \"$path\" || true")+ }+ }+ { print }+ END { if (count != 1) exit 1 }+ ' "$FIXTURE_ROOT/aggregate-original.sh" > "$FIXTURE_ROOT/aggregate-$mutation.sh"; then+ fail "cannot construct $mutation mutation; update its injection site"+ continue+ fi+ if verify_aggregate "$mutation"; then+ fail "contract accepted $mutation despite preserved names/counter/PASS"+ else+ pass "contract rejects $mutation despite preserved names/counter/PASS"+ fidone-if grep -Fq 'live drift and owner repair' "$WEBHOOK" &&- grep -Fq 'Signed webhook route recovered' "$WEBHOOK" &&- grep -Fq 'stayed resourceVersion-stable' "$WEBHOOK"; then- pass "webhook E2E proves all three policies, owner repair, route recovery, and no churn"-else- fail "webhook E2E lacks a required convergence signal"-fiexit "$FAIL"diff --git a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh b/scripts/tests/test-wrc-networkpolicy-probe-contract.shnew file mode 100755index 000000000..14e0c3cc4--- /dev/null+++ b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh@@ -0,0 +1,231 @@+#!/usr/bin/env bash+# Contract tests execute the real HTTP probe and observation decisions. Only+# Kubernetes transport, the remote nc process, and elapsed time are fixtures.+# Functions are dispatched through expect_pass/expect_reject and the sourced helper.+# shellcheck disable=SC2329+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-np-probe-contract.XXXXXX")"+trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT+# shellcheck source=../e2e/_lib/wrc-networkpolicy-convergence.sh+source "$ROOT/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"++FAIL=0+POLL_INTERVAL=1+export E2E_PROBE_TIMEOUT=2+export PROBE_SCENARIO=http+export PROBE_SEQUENCE_COUNTER="$FIXTURE_ROOT/nc-counter"+PROBE_TRANSPORT=http+OBSERVATION_SCENARIO=clean+ok() { :; }+fail() { printf '%s\n' "$*" >&2; }++expect_pass() {+ local label=$1+ shift+ if "$@" > "$FIXTURE_ROOT/result" 2>&1; then+ printf 'PASS: %s\n' "$label"+ else+ printf 'FAIL: %s\n' "$label" >&2+ cat "$FIXTURE_ROOT/result" >&2+ FAIL=1+ fi+}++expect_reject() {+ local label=$1+ shift+ if "$@" > "$FIXTURE_ROOT/result" 2>&1; then+ printf 'FAIL: accepted %s\n' "$label" >&2+ cat "$FIXTURE_ROOT/result" >&2+ FAIL=1+ else+ printf 'PASS: rejects %s\n' "$label"+ fi+}++mkdir -p "$FIXTURE_ROOT/tools" "$FIXTURE_ROOT/missing-tools"+for tool in grep base64 tr mktemp rm; do+ ln -s "$(command -v "$tool")" "$FIXTURE_ROOT/tools/$tool"+done+cat > "$FIXTURE_ROOT/tools/nc" <<'NC'+#!/bin/sh+case "$PROBE_SCENARIO" in+ timeout-then-http|http-then-timeout)+ read -r count < "$PROBE_SEQUENCE_COUNTER"+ count=$((count + 1))+ printf '%s\n' "$count" > "$PROBE_SEQUENCE_COUNTER"+ if { [ "$PROBE_SCENARIO" = timeout-then-http ] && [ "$count" -eq 1 ]; } ||+ { [ "$PROBE_SCENARIO" = http-then-timeout ] && [ "$count" -gt 1 ]; }; then+ printf 'nc: connect timed out\n' >&2+ exit 1+ fi+ printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal'+ exit 0 ;;+esac+case "$PROBE_SCENARIO" in+ http) printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal' ;;+ http-403) printf 'HTTP/1.0 403 Forbidden\r\n\r\nfixture-auth-denied' ;;+ timeout) printf 'nc: connect timed out\n' >&2; exit 1 ;;+ refused) printf 'nc: Connection refused\n' >&2; exit 1 ;;+ empty) exit 0 ;;+ malformed) printf 'not an HTTP response' ;;+ *) exit 99 ;;+esac+NC+chmod +x "$FIXTURE_ROOT/tools/nc"++log_event() {+ printf '%s %s\n' "$1" "$2" >> "$FIXTURE_ROOT/logs"+}++noop_event='{"msg":"network policy unchanged; skipping update","namespace":"sandbox-recipes","policy":"policy","family":"workload-ingress"}'+complete_event='{"msg":"recipe reconciliation completed","recipe":"recipe","namespace":"sandbox-recipes","uid":"recipe-uid","generation":2,"phase":"active","requeueAfterMs":0}'+write_event='{"msg":"network policy replaced","namespace":"sandbox-recipes","policy":"policy","family":"wrong-family"}'++kctl() {+ if [ "$1" = exec ]; then+ printf 'exec\n' >> "$FIXTURE_ROOT/exec-calls"+ case "$PROBE_TRANSPORT" in+ exec-forbidden) printf 'forbidden: pods/exec\n' >&2; return 13 ;;+ empty-exec) return 0 ;;+ bad-nonce) printf 'WRC_NP_BEGIN|foreign\nWRC_NP_END|foreign|timeout|\n'; return 0 ;;+ malformed-frame) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|unknown|\n' "${10}" "${10}"; return 0 ;;+ malformed-base64) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|http|!!!!\n' "${10}" "${10}"; return 0 ;;+ malformed-http) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|http|bm90IEhUVFA=\n' "${10}" "${10}"; return 0 ;;+ esac+ while [ "$1" != -- ]; do shift; done+ shift+ [ "$1" = sh ] || return 99+ shift+ if [ "$PROBE_TRANSPORT" = missing-tool ]; then+ PATH="$FIXTURE_ROOT/missing-tools" /bin/sh "$@"+ else+ PATH="$FIXTURE_ROOT/tools" /bin/sh "$@"+ fi+ return+ fi+ case "$1:$2" in+ get:deployment) printf '%s\n' '{"spec":{"replicas":1,"selector":{"matchLabels":{"app":"wrc"}}}}' ;;+ get:pods) cat "$FIXTURE_ROOT/pods" ;;+ logs:*) cat "$FIXTURE_ROOT/logs" ;;+ get:networkpolicy) cat "$FIXTURE_ROOT/policy" ;;+ get:workflowrecipe)+ if [[ "$*" == *'jsonpath='* ]]; then+ cat "$FIXTURE_ROOT/generation"+ else+ printf '%s\n' '{"metadata":{"uid":"recipe-uid","generation":1}}'+ fi ;;+ patch:workflowrecipe) printf '2\n' > "$FIXTURE_ROOT/generation" ;;+ *) printf 'unexpected fixture transport: %s\n' "$*" >&2; return 99 ;;+ esac+}++# A fixed cursor and advancing elapsed clock keep this matrix bounded without+# sleeping or weakening the production minimum observation window of one second.+# Unsetting Bash's special SECONDS removes its wall-clock behaviour; assigning+# zero alone would leave a timing race between the real clock and fixture ticks.+unset SECONDS+SECONDS=0+wrc_now() { printf '2026-09-04T00:00:02.000000000Z\n'; }+sleep() {+ SECONDS=$((SECONDS + $1))+ printf 'tick\n' >> "$FIXTURE_ROOT/ticks"+ if [ "$OBSERVATION_SCENARIO" = delayed-write ]; then+ log_event '2026-09-04T00:00:04.000000000Z' "$write_event"+ elif [ "$OBSERVATION_SCENARIO" = observer-changed ]; then+ printf '%s\n' '{"items":[{"metadata":{"name":"wrc","uid":"pod-uid"},"status":{"phase":"Running","containerStatuses":[{"name":"wrc","ready":true,"containerID":"container-new","restartCount":1}]}}]}' > "$FIXTURE_ROOT/pods"+ fi+}++probe_allowed() {+ wrc_assert_http_allowed 'fixture route' sandbox-recipes pod fixture 8080 fixture-business-signal+}+probe_blocked() {+ wrc_assert_http_blocked 'fixture isolation' sandbox-recipes pod fixture 8080+}++expect_pass 'HTTP probe executes remote tools and validates the application response' probe_allowed+expect_reject 'HTTP response as NetworkPolicy denial' probe_blocked+PROBE_SCENARIO=http-403+expect_reject 'HTTP authorization rejection as NetworkPolicy denial' probe_blocked+PROBE_SCENARIO=timeout+expect_pass 'completed remote connection timeout as a network observation' probe_blocked+expect_reject 'timeout as an allowed route' probe_allowed+for PROBE_SCENARIO in empty malformed refused; do+ expect_reject "remote $PROBE_SCENARIO response as NetworkPolicy denial" probe_blocked+done+PROBE_SCENARIO=http+for PROBE_TRANSPORT in exec-forbidden missing-tool empty-exec bad-nonce malformed-frame malformed-base64 malformed-http; do+ expect_reject "$PROBE_TRANSPORT probe envelope" wrc_http_probe sandbox-recipes pod fixture 8080+done+PROBE_TRANSPORT=http++probe_sequence() {+ local scenario=$1 assertion=$2 count+ PROBE_SCENARIO=$scenario+ printf '0\n' > "$PROBE_SEQUENCE_COUNTER"+ "$assertion" || return 1+ read -r count < "$PROBE_SEQUENCE_COUNTER"+ [ "$count" -eq 2 ]+}++executor_aborts_once() {+ local assertion=$1 before=$SECONDS count+ PROBE_TRANSPORT=exec-forbidden+ : > "$FIXTURE_ROOT/exec-calls"+ if "$assertion"; then return 1; fi+ count="$(wc -l < "$FIXTURE_ROOT/exec-calls")"+ [ "$count" -eq 1 ] && [ "$SECONDS" -eq "$before" ]+}++expect_pass 'allowed route waits through timeout then validates HTTP after CNI propagation' \+ probe_sequence timeout-then-http probe_allowed+expect_pass 'denied route waits through HTTP then observes timeout after CNI propagation' \+ probe_sequence http-then-timeout probe_blocked+expect_pass 'allowed assertion aborts once on exec failure without waiting/retrying' executor_aborts_once probe_allowed+expect_pass 'denied assertion aborts once on exec failure without waiting/retrying' executor_aborts_once probe_blocked+PROBE_SCENARIO=http+PROBE_TRANSPORT=http++observe() {+ local scenario=$1+ OBSERVATION_SCENARIO=$scenario+ : > "$FIXTURE_ROOT/logs"+ : > "$FIXTURE_ROOT/ticks"+ printf '1\n' > "$FIXTURE_ROOT/generation"+ printf '%s\n' '{"metadata":{"uid":"policy-uid","resourceVersion":"1"}}' > "$FIXTURE_ROOT/policy"+ printf '%s\n' '{"items":[{"metadata":{"name":"wrc","uid":"pod-uid"},"status":{"phase":"Running","containerStatuses":[{"name":"wrc","ready":true,"containerID":"container-one","restartCount":0}]}}]}' > "$FIXTURE_ROOT/pods"+ wrc_begin_np_observation || return 1+ wrc_track_np sandbox-recipes policy workload-ingress || return 1+ if [ "$scenario" = pretrigger-rv-drift ]; then+ printf '%s\n' '{"metadata":{"uid":"policy-uid","resourceVersion":"2"}}' > "$FIXTURE_ROOT/policy"+ fi+ case "$scenario" in+ write-before)+ log_event '2026-09-04T00:00:02.100000000Z' "$write_event"+ log_event '2026-09-04T00:00:03.000000000Z' "$noop_event" ;;+ write-after)+ log_event '2026-09-04T00:00:03.000000000Z' "$noop_event"+ log_event '2026-09-04T00:00:03.100000000Z' "$write_event" ;;+ wrong-family)+ log_event '2026-09-04T00:00:03.000000000Z' "${noop_event/workload-ingress/oauth-broker-egress}" ;;+ missing-witness) ;;+ old-logs) log_event '2026-09-04T00:00:01.999999999Z' "$noop_event" ;;+ *) log_event '2026-09-04T00:00:03.000000000Z' "$noop_event" ;;+ esac+ log_event '2026-09-04T00:00:03.200000000Z' "$complete_event"+ wrc_trigger_recipe_reconcile sandbox-recipes recipe 3 || return 1+ SECONDS=0+ wrc_assert_np_observation_clean 1 3 || return 1+ # An initial no-op is insufficient: success must span the full window.+ [ -s "$FIXTURE_ROOT/ticks" ]+}++expect_pass 'complete no-write window with an exact active reconcile and stable observer/UID/RV' observe clean+for scenario in write-before write-after delayed-write wrong-family missing-witness old-logs observer-changed pretrigger-rv-drift; do+ expect_reject "$scenario despite a completed parent reconcile" observe "$scenario"+done++exit "$FAIL"diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex a099d73c4..fde522079 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.345",+ "version": "0.1.346","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.345",+ "version": "0.1.346","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex 474d7ac77..af9e20fcd 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.345",+ "version": "0.1.346","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.test.tsindex e708077f9..c42e1cfb6 100644--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts@@ -133,6 +133,7 @@ describe('NetworkPolicy live convergence', () => {it('makes workload-egress metadata drift bypass its temporal prefilter', () => {const desired = policy()const live = structuredClone(desired)+ desired.metadata!.annotations = { 'clerum.io/egress-fqdn-resolved-at': 'new-timestamp' }live.metadata!.annotations = { 'clerum.io/egress-fqdn-resolved-at': 'old-timestamp' }expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(true)@@ -157,6 +158,21 @@ describe('NetworkPolicy live convergence', () => {expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)})+ it('compares stable workload-egress annotations and removal of owned state', () => {+ const desired = policy()+ desired.metadata!.annotations = { 'clerum.io/test-contract': 'current' }+ const live = structuredClone(desired)+ live.metadata!.annotations!['clerum.io/test-contract'] = 'stale'+ expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)++ live.metadata!.annotations = {+ ...desired.metadata!.annotations,+ 'clerum.io/egress-fqdn-state': 'stale',+ }+ expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)+ expect(networkPolicyMatchesDesired(desired, live)).toBe(false)+ })+it('repairs a stale or missing gateway owner only for the same WRC recipe identity', () => {const desired = gatewayPolicy('uid-new')const stale = gatewayPolicy('uid-old')diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.tsindex ee5ad05d7..d37c830fd 100644--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts@@ -1,4 +1,9 @@import * as k8s from '@kubernetes/client-node'+import {+ RESOLVED_AT_ANNOTATION,+ STATE_ANNOTATION,+ TARGETS_ANNOTATION,+} from '@clerum/network-policy-core'import { SPEC_HASH_ANNOTATION } from './specHash'export type NetworkPolicyFamily =@@ -41,6 +46,64 @@ const GATEWAY_IDENTITY_LABELS = ['clerum.io/webhook-gateway',] as const+// These keys remain WRC-owned when external egress is removed from desired.+// Other annotation namespaces are not ownership boundaries: preserve every+// unauthored key, including keys authored by other clerum.io controllers.+const EGRESS_STATE_ANNOTATIONS = new Set([+ RESOLVED_AT_ANNOTATION,+ STATE_ANNOTATION,+ TARGETS_ANNOTATION,+])++function projectOwnedAnnotations(+ desired: Record<string, string> | undefined,+ existing: Record<string, string> | undefined,+ temporal = false+): Record<string, string> {+ const keys = new Set([...Object.keys(desired ?? {}), ...EGRESS_STATE_ANNOTATIONS])+ return Object.fromEntries(+ [...keys].flatMap(key => {+ if (key === SPEC_HASH_ANNOTATION) return []+ // The accumulator owns renewal cadence for present temporal state. Its+ // removal still has to reach apply, even when the enforced rules match.+ if (temporal && EGRESS_STATE_ANNOTATIONS.has(key) && desired?.[key] !== undefined) return []+ return existing?.[key] === undefined ? [] : [[key, existing[key]]]+ })+ )+}++/** Build a PUT from the same live snapshot used for ownership and drift checks. */+export function buildNetworkPolicyReplacement(+ desired: k8s.V1NetworkPolicy,+ existing: k8s.V1NetworkPolicy+): k8s.V1NetworkPolicy {+ const replacement = structuredClone(desired)+ const liveMetadata = existing.metadata ?? {}+ const annotations = { ...liveMetadata.annotations }+ for (const key of EGRESS_STATE_ANNOTATIONS) delete annotations[key]+ Object.assign(annotations, desired.metadata?.annotations)+ delete annotations[SPEC_HASH_ANNOTATION]+ replacement.metadata = {+ ...replacement.metadata,+ labels: { ...liveMetadata.labels, ...replacement.metadata?.labels },+ annotations,+ ...(liveMetadata.finalizers?.length || replacement.metadata?.finalizers?.length+ ? {+ finalizers: [+ ...new Set([+ ...(liveMetadata.finalizers ?? []),+ ...(replacement.metadata?.finalizers ?? []),+ ]),+ ],+ }+ : {}),+ resourceVersion: liveMetadata.resourceVersion,+ }+ // ownerReferences intentionally come only from desired after the ownership+ // veto. Foreign lifecycle owners must not be adopted, retained, or erased.+ return replacement+}+function canonicalizeValue(value: unknown): unknown {if (Array.isArray(value)) return value.map(canonicalizeValue)if (value && typeof value === 'object') {@@ -54,13 +117,12 @@ function canonicalizeValue(value: unknown): unknown {function projectAuthoredMap(desired: Record<string, string> | undefined,- existing: Record<string, string> | undefined,- excludedKeys: ReadonlySet<string> = new Set()+ existing: Record<string, string> | undefined): Record<string, string> {return Object.fromEntries(- Object.keys(desired ?? {})- .filter(key => !excludedKeys.has(key))- .flatMap(key => (existing?.[key] === undefined ? [] : [[key, existing[key]]]))+ Object.keys(desired ?? {}).flatMap(key =>+ existing?.[key] === undefined ? [] : [[key, existing[key]]]+ ))}@@ -77,10 +139,9 @@ function projectNetworkPolicyForComparison(name: candidateMetadata.name,namespace: candidateMetadata.namespace,labels: projectAuthoredMap(desiredMetadata.labels, candidateMetadata.labels),- annotations: projectAuthoredMap(+ annotations: projectOwnedAnnotations(desiredMetadata.annotations,- candidateMetadata.annotations,- new Set([SPEC_HASH_ANNOTATION])+ candidateMetadata.annotations),...(desiredMetadata.ownerReferences?.length? { ownerReferences: candidateMetadata.ownerReferences }@@ -145,8 +206,8 @@ export function networkPolicyMatchesDesired(/*** Compare the stable identity metadata that the workload-egress prefilter must- * never hide. Temporal DNS/state annotations are intentionally excluded: their- * persistence cadence is decided separately by the accumulator, while labels,+ * never hide. Present temporal DNS/state values follow the accumulator's+ * persistence cadence; retired state keys, stable annotations, labels,* lifecycle owner and a terminating object must still reach the live apply.*/export function networkPolicyMetadataMatchesDesired(@@ -164,6 +225,11 @@ export function networkPolicyMetadataMatchesDesired(name: policy.metadata?.name,namespace: policy.metadata?.namespace,labels: projectAuthoredMap(desired.metadata?.labels, policy.metadata?.labels),+ annotations: projectOwnedAnnotations(+ desired.metadata?.annotations,+ policy.metadata?.annotations,+ true+ ),ownerReferences: desired.metadata?.ownerReferences?.length? policy.metadata?.ownerReferences: undefined,@@ -219,6 +285,24 @@ export function classifyNetworkPolicyOwnership(const desiredOwner = desiredOwners[0]if (!desiredOwner.uid) return { kind: 'conflict', reason: 'desired-owner-uid-missing' }+ // Admission metadata may coexist, but an additional lifecycle owner changes+ // garbage-collection authority. Only explicitly desired non-controller refs+ // are permitted; never silently drop an unexpected owner during replacement.+ const desiredOtherOwners = (desired.metadata?.ownerReferences ?? []).filter(+ owner => owner.controller !== true+ )+ const existingOtherOwners = (existing.metadata?.ownerReferences ?? []).filter(+ owner => owner.controller !== true+ )+ if (+ existingOtherOwners.length !== desiredOtherOwners.length ||+ existingOtherOwners.some(+ owner => !desiredOtherOwners.some(wanted => exactOwnerMatch(wanted, owner))+ )+ ) {+ return { kind: 'conflict', reason: 'owner-reference-mismatch' }+ }+if (existingOwners.length === 1 && exactOwnerMatch(desiredOwner, existingOwners[0])) {return { kind: 'owned' }}diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 7980d176b..11747dabe 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -859,6 +859,134 @@ describe('WorkflowRecipeReconciler', () => {},})+ const families: NetworkPolicyFamily[] = [+ 'ui-ingress',+ 'workload-ingress',+ 'workload-egress',+ 'internal-dependency',+ 'oauth-broker-egress',+ 'webhook-gateway',+ ]+ const policyForFamily = (family: NetworkPolicyFamily) =>+ family === 'webhook-gateway'+ ? gatewayPolicy()+ : family === 'internal-dependency'+ ? intdepPolicy()+ : plainPolicy()++ it.each(families)(+ 'preserves external metadata while repairing real %s spec drift',+ async family => {+ const desired = policyForFamily(family)+ desired.metadata!.labels!['clerum.io/test-contract'] = 'current'+ desired.metadata!.annotations = {+ 'clerum.io/test-contract': 'current',+ [SPEC_HASH]: 'obsolete',+ }+ const live = livePolicy(desired)+ live.metadata!.labels!['clerum.io/test-contract'] = 'stale'+ live.metadata!.labels!['admission.example/tier'] = 'audited'+ live.metadata!.annotations = {+ 'admission.example/audit': 'keep',+ 'clerum.io/test-contract': 'stale',+ [SPEC_HASH]: 'legacy',+ 'clerum.io/egress-fqdn-state': 'stale-state',+ 'clerum.io/egress-fqdn-targets': 'stale-targets',+ 'clerum.io/egress-fqdn-resolved-at': 'stale-time',+ }+ live.metadata!.finalizers = ['admission.example/cleanup']+ live.spec!.podSelector = {}+ const desiredBefore = clone(desired)+ const liveBefore = clone(live)+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await apply(desired, family)++ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const body = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(body.metadata.labels).toEqual({+ ...desired.metadata!.labels,+ 'admission.example/tier': 'audited',+ })+ expect(body.metadata.annotations).toEqual({+ 'admission.example/audit': 'keep',+ 'clerum.io/test-contract': 'current',+ })+ expect(body.metadata.finalizers).toEqual(['admission.example/cleanup'])+ expect(body.metadata.ownerReferences).toEqual(desired.metadata!.ownerReferences)+ expect(body.metadata.resourceVersion).toBe('9')+ expect(body.spec).toEqual(desired.spec)+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(desired).toEqual(desiredBefore)+ expect(live).toEqual(liveBefore)+ }+ )++ it.each(families)(+ 'performs zero create/replace calls for equivalent %s with external metadata',+ async family => {+ const desired = policyForFamily(family)+ const live = livePolicy(desired)+ live.metadata!.labels!['admission.example/tier'] = 'audited'+ live.metadata!.annotations = { 'admission.example/audit': 'keep', [SPEC_HASH]: 'legacy' }+ live.metadata!.finalizers = ['admission.example/cleanup']+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await apply(desired, family)++ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ }+ )++ it('vetoes an additional non-controller owner instead of removing it on gateway replace', async () => {+ const desired = gatewayPolicy()+ const live = livePolicy(desired)+ live.metadata!.ownerReferences!.push({+ apiVersion: 'v1',+ kind: 'ConfigMap',+ name: 'foreign',+ uid: 'foreign',+ })+ live.spec!.podSelector = {}+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)++ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({+ name: 'NetworkPolicyOwnershipConflictError',+ })+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })++ it('removes retired workload-egress state once without churning external metadata', async () => {+ const desired = plainPolicy()+ desired.metadata!.name = 'wl-egress-r-w'+ const live = livePolicy(desired, {+ annotations: {+ 'admission.example/audit': 'keep',+ 'clerum.io/egress-fqdn-state': 'retired',+ 'clerum.io/egress-fqdn-targets': 'retired',+ 'clerum.io/egress-fqdn-resolved-at': 'retired',+ },+ })+ const priv = reconciler as unknown as {+ egressWriteNeeded: (existing: k8s.V1NetworkPolicy, desired: k8s.V1NetworkPolicy) => boolean+ }+ expect(priv.egressWriteNeeded(live, desired)).toBe(true)+ await apply(desired, 'workload-egress', live)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)+ const updated = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body+ expect(updated.metadata.annotations).toEqual({ 'admission.example/audit': 'keep' })+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()++ expect(priv.egressWriteNeeded(updated, desired)).toBe(false)+ await apply(desired, 'workload-egress', updated)+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })+function captureNetworkPolicyLogs(): {entries: Array<Record<string, unknown>>restore: () => void@@ -1080,7 +1208,7 @@ describe('WorkflowRecipeReconciler', () => {expect(warning).toEqual(expect.objectContaining({level: 'warn',- errorName: 'UnknownError',+ errorName: 'NetworkPolicyInfraError',errorCode: 500,retryable: true,})@@ -3056,6 +3184,136 @@ describe('WorkflowRecipeReconciler', () => {)})+ describe('workload egress pre-read infrastructure failures', () => {+ it.each([+ { workflow: true, code: 503 },+ { workflow: true, code: 'ECONNRESET' },+ { workflow: false, code: 503 },+ { workflow: false, code: 'ECONNRESET' },+ { workflow: true, code: 403 },+ { workflow: false, code: 403 },+ ])('classifies $code after successful DNS (workflow=$workflow)', async ({ workflow, code }) => {+ const recipe = makeRecipe({+ spec: {+ workloads: [+ {+ id: 'worker',+ type: 'deployment',+ image: 'worker:latest',+ port: 8080,+ egressBindings: [{ dns: 'api.example.com', port: 443 }],+ },+ ],+ ...(workflow ? { steps: [{ id: 'run', run: snippetRun() }] } : {}),+ },+ ...(!workflow+ ? {+ status: {+ phase: 'active' as const,+ message: 'Healthy before read failure',+ workloads: [],+ },+ }+ : {}),+ })+ const lookup = vi.fn(async () => ({+ kind: 'ok' as const,+ ipv4: ['93.184.216.10'],+ ipv6: [],+ ttlSeconds: 300,+ }))+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {+ fqdnLookup: lookup,+ })+ const workflowReconcile = vi.fn()+ ;(rec as unknown as { workflowReconciler: unknown }).workflowReconciler = {+ reconcile: workflowReconcile,+ validateWorkflowSpec: () => undefined,+ }+ const failure = { code }+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {+ if (name === 'wl-egress-test-recipe-worker') {+ expect(lookup).toHaveBeenCalled()+ throw failure+ }+ throw { code: 404 }+ })++ const result = await rec.reconcile(recipe)++ expect(lookup).toHaveBeenCalledTimes(1)+ if (code === 403) {+ expect(result.phase).toBe('failed')+ expect(result.requeueAfterMs).toBeUndefined()+ } else {+ expect(result.phase).toBe(workflow ? 'degraded' : 'active')+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)+ if (!workflow) expect(result.skipStatusPatch).toBe(true)+ }+ expect(workflowReconcile).not.toHaveBeenCalled()+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)+ })+ })++ describe('reconciliation completion receipt', () => {+ it.each(['success', 'reconcile-throw', 'projection-throw'] as const)(+ 'emits completion only after the full pipeline returns (%s)',+ async outcome => {+ const recipe = makeRecipe()+ recipe.metadata.generation = 7+ const previousLevel = process.env.LOG_LEVEL+ process.env.LOG_LEVEL = 'info'+ const receipts: Array<Record<string, unknown>> = []+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {+ for (const line of String(chunk).split('\n')) {+ if (!line.startsWith('{')) continue+ const record = JSON.parse(line)+ if (record.msg === 'recipe reconciliation completed') receipts.push(record)+ }+ return true+ }) as typeof process.stdout.write)+ const priv = reconciler as unknown as {+ reconcileInternal: () => Promise<unknown>+ projectPluginWorkloadSdk: () => unknown+ }+ const failure = new Error('pipeline failure')+ const injected =+ outcome === 'reconcile-throw'+ ? vi.spyOn(priv, 'reconcileInternal').mockRejectedValue(failure)+ : outcome === 'projection-throw'+ ? vi.spyOn(priv, 'projectPluginWorkloadSdk').mockImplementation(() => {+ throw failure+ })+ : undefined+ try {+ if (outcome === 'success') {+ const result = await reconciler.reconcile(recipe)+ expect(result.phase).toBe('active')+ expect(receipts).toEqual([+ expect.objectContaining({+ recipe: 'test-recipe',+ namespace: 'sandbox-recipes',+ uid: 'uid-123',+ generation: 7,+ phase: 'active',+ requeueAfterMs: 0,+ }),+ ])+ } else {+ await expect(reconciler.reconcile(recipe)).rejects.toBe(failure)+ expect(receipts).toEqual([])+ }+ } finally {+ injected?.mockRestore()+ write.mockRestore()+ if (previousLevel === undefined) delete process.env.LOG_LEVEL+ else process.env.LOG_LEVEL = previousLevel+ }+ }+ )+ })+it('fails non-transport external egress in required mode when cluster enforcement is not confirmed', async () => {const recipe = makeRecipe({spec: {diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex bb72bea0e..e364bb611 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -91,6 +91,7 @@ import {} from './mcpDelegation'import {type NetworkPolicyFamily,+ buildNetworkPolicyReplacement,decideNetworkPolicyConvergence,networkPolicyMetadataMatchesDesired,} from './networkPolicyConvergence'@@ -1425,6 +1426,14 @@ export class WorkflowRecipeReconciler {// both consumers treat as "no SDK opinion this pass".const result = await this.reconcileInternal(recipe)result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)+ createLogger('wrc', recipe.metadata.name).info('recipe reconciliation completed', {+ recipe: recipe.metadata.name,+ namespace: recipe.metadata.namespace,+ uid: recipe.metadata.uid,+ generation: recipe.metadata.generation,+ phase: result.phase,+ requeueAfterMs: result.requeueAfterMs ?? 0,+ })return result}@@ -4759,6 +4768,14 @@ export class WorkflowRecipeReconciler {return await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })} catch (error: unknown) {if (getErrorCode(error) === 404) return null+ if (isRetryableInfraError(error)) {+ const safe = this.safeInfrastructureErrorFields(error)+ throw new NetworkPolicyInfraError(+ `NetworkPolicy "${name}" in ${namespace} read temporarily unavailable (${safe.errorCode ?? safe.errorName})`,+ error,+ safe.errorCode+ )+ }throw error}}@@ -5011,12 +5028,12 @@ export class WorkflowRecipeReconciler {`NetworkPolicy "${policyName}" in ${namespace} has no resourceVersion`)}- desired.metadata.resourceVersion = resourceVersion+ const replacement = buildNetworkPolicyReplacement(desired, existing)try {await this.networkingApi.replaceNamespacedNetworkPolicy({name: policyName,namespace,- body: desired,+ body: replacement,})log.info('network policy replaced', {...fields,@@ -5073,22 +5090,11 @@ export class WorkflowRecipeReconciler {for (let attempt = 1; attempt <= 2; attempt += 1) {try {return {- existing: await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),+ existing: await this.readNetworkPolicyOrNull(name, namespace),recoveredFromReadFailure: attempt > 1,}} catch (error: unknown) {- if (getErrorCode(error) === 404) {- return { existing: null, recoveredFromReadFailure: attempt > 1 }- }if (attempt === 2) {- if (isRetryableInfraError(error)) {- const safe = this.safeInfrastructureErrorFields(error)- throw new NetworkPolicyInfraError(- `NetworkPolicy "${name}" in ${namespace} read temporarily unavailable (${safe.errorCode ?? safe.errorName})`,- error,- safe.errorCode- )- }throw error}log.warn('network policy read unavailable; retrying once', {