Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): harden NetworkPolicy recovery and validation

Alfredo Lopez475cdbfe0094
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 28f1c64e3..36a43ec50 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -546,7 +546,7 @@ jobs:
run: bash scripts/tests/test-e2e-vitest-node-unit-gate.sh
- name: Test WRC NetworkPolicy live-convergence E2E contract
- run: bash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
+ run: make test-wrc-networkpolicy-contracts
- name: Test GFS real-Postgres Minikube runner contract
run: bash scripts/tests/test-gfs-real-pg-minikube-gate.sh
diff --git a/Makefile b/Makefile
index 6813b3f68..3f30839ec 100644
--- a/Makefile
+++ b/Makefile
@@ -1083,7 +1083,7 @@ test-contracts: test-e2e-deps ## Run contract tests only
test-e2e-bash: ## Run bash-based E2E suites (scripts/e2e/*.sh)
@echo "Running bash E2E suites..."
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-workflow-runtime-gate.sh
- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-workflow-backend-compat.sh
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-sfs-legacy-job-cleanup.sh
KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-sfs-security.sh
@@ -1096,12 +1096,18 @@ test-e2e-workflow-runtime: ## Run workflow runtime E2E gate
.PHONY: test-e2e-wrc-internal-dependency-networkpolicy
test-e2e-wrc-internal-dependency-networkpolicy: ## Run issue #485 WRC internal-dependency NetworkPolicy E2E gate
@echo "Running WRC internal-dependency NetworkPolicy E2E gate..."
- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
.PHONY: test-e2e-wrc-networkpolicy-live-convergence
test-e2e-wrc-networkpolicy-live-convergence: ## Run all PR #580 WRC NetworkPolicy family, route, repair, owner, and no-churn E2E gates
@echo "Running WRC NetworkPolicy live-convergence E2E gate..."
- KUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
+ KUBECONTEXT="$(E2E_KUBECONTEXT)" MINIKUBE_PROFILE="$(E2E_KUBECONTEXT)" CONTROL_API_REAL_PG_CONTEXT="$(E2E_KUBECONTEXT)" bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
+
+.PHONY: test-wrc-networkpolicy-contracts
+test-wrc-networkpolicy-contracts: ## Run hermetic WRC NetworkPolicy runner, probe, observation, and fixture contracts
+ bash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
+ bash scripts/tests/test-wrc-networkpolicy-probe-contract.sh
+ bash scripts/tests/test-wrc-fixtures.sh
.PHONY: test-e2e-codex-subscription-network-boundary
test-e2e-codex-subscription-network-boundary: ## Codex LLM proxy NetworkPolicy boundary (exit 3 before deploy)
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 0a123f28d..80273e931 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -329,6 +329,9 @@ it cannot hide later drift.
WRC logger. In steady state expect `network policy unchanged; skipping update` or
`network policy egress set unchanged; skipping live apply` with `policy`, `namespace`
and `family` fields. Re-baseline alerts keyed on the old `Updated …` console line.
+ `recipe reconciliation completed` records recipe UID, generation, outcome and retry
+ delay only after the reconciliation result is ready; entering a reconcile is not its
+ completion signal.
- **NetworkPolicy drift is auto-reverted.** A live change to `podSelector`,
`policyTypes`, `ingress`, `egress`, WRC-authored metadata or the gateway controller
owner forces one optimistic-concurrency replace using the `resourceVersion` from the
@@ -338,6 +341,11 @@ it cannot hide later drift.
whose winner disappears, missing live `resourceVersion`, replace 404/409 or transient
API failure re-enqueues the parent with bounded backoff. A status-only update is not
treated as the retry trigger.
+- **Metadata ownership applies to writes too.** Replacing a policy preserves external
+ labels, annotations and finalizers from the validated snapshot. Desired WRC keys win;
+ retired egress state and the legacy spec-hash are removed. Additional foreign lifecycle
+ owners are rejected, not silently dropped or adopted. The workload-egress pre-read uses
+ the same transient-error classification as the downstream apply path.
- **Other spec-hash-managed objects retain their existing drift contract.** For example,
a manually scaled recipe Deployment is not reverted until the desired manifest changes;
its replica-health drift remains visible in recipe status.
@@ -352,7 +360,8 @@ it cannot hide later drift.
### WRC NetworkPolicy live-convergence E2E
NetworkPolicy changes must run the aggregate gate against an isolated development
-cluster with an explicit context:
+cluster with an explicit context. Its Make target acquires the existing profile mutation
+lease; individual suites require that same inherited lease:
```bash
E2E_KUBECONTEXT=<branch-owned-context> \
@@ -363,10 +372,11 @@ The aggregate fails unless all four executable suites run. Together they cover e
NetworkPolicy family routed through the live-convergence helper:
- `ui-ingress`, `workload-ingress` and `workload-egress`: correctly configured UI→API
- and API→sibling traffic, unlabelled negative controls, forced live-spec drift, a real
+ and API→sibling traffic, isolated ingress and undeclared-sibling egress controls,
+ forced live-spec drift, a real
parent recipe reconcile, repair, a terminating-policy race that self-heals through the
scheduled retry without a second parent event, and a post-repair no-op witness plus
- resourceVersion stability window;
+ pre-trigger UID/resourceVersion baseline and full observation window;
- `internal-dependency`: inferred source→backend traffic, an unlabelled denied pod,
two-sided policy repair and no-churn;
- `oauth-broker-egress`: opted-in workload→Control API reachability, an unlabelled
@@ -374,13 +384,30 @@ NetworkPolicy family routed through the live-convergence helper:
- `webhook-gateway`: all three policies, a signed public webhook business signal,
spec drift, stale/missing owner repair, route recovery and no-churn.
-Every no-churn assertion requires both a policy/family-bound structured WRC no-op log and
-an unchanged `resourceVersion`; observing only the generic parent-reconcile log is not
-sufficient. The public CI runner cannot provide this Calico-backed cluster. It therefore
-runs the fail-loud composition/syntax contract, while the real dataplane result is
-recorded by the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a
-missing family suite, incomplete cleanup or zero executed suites are failures, never
-skips.
+Negative controls first prove reachability from the same probe with a narrow temporary
+permission, remove only that permission, observe a remotely executed connection timeout,
+and prove reachability again. Complementary permissions isolate ingress from egress.
+An exec, authorization, missing-tool or malformed-response error fails the test instead
+of counting as a policy denial. Condition-based polling accommodates CNI propagation.
+
+Every no-churn observation begins before the parent trigger. It requires the exact
+UID/generation completion receipt, every policy/family no-op witness, unchanged policy
+UID/resourceVersion, stable controller replica/container identities, and no observed
+policy write during the entire window. A no-op followed by a write fails even when the
+resourceVersion does not change. Exact API-call counts are asserted separately in unit
+tests; the runtime logs are not a substitute for exhaustive Kubernetes audit accounting.
+
+Fixtures use unique run IDs, collision-safe creation, UID-preconditioned deletion and
+owned dynamic port-forwards. Cleanup never enrolls an existing sample by name, and a
+cleanup failure fails the suite. `E2E_KEEP_RESOURCES=1` explicitly retains run
+resources for inspection, while still stopping owned port-forwards.
+
+The public CI runner cannot provide this Calico-backed cluster. It runs
+`make test-wrc-networkpolicy-contracts`: the actual aggregate against instrumented child
+processes, every child failure/missing-child case, helper fault injection, and fixture
+ownership/cleanup tests. Skipping the child invocation or swallowing a child error fails
+this contract. The real dataplane result remains a separate local T2/pre-merge evidence
+lane. Missing prerequisites, incomplete observations and zero execution are failures.
### Unit tests (vitest + @testing-library/react)
diff --git a/scripts/e2e/_lib/wrc-fixtures.sh b/scripts/e2e/_lib/wrc-fixtures.sh
new file mode 100755
index 000000000..36724e6ac
--- /dev/null
+++ b/scripts/e2e/_lib/wrc-fixtures.sh
@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+# Run-owned WRC fixtures. The caller must first enter the branch mutation lease.
+# shellcheck disable=SC1091
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../../minikube" && pwd)/port-forward-owner.sh"
+
+wrc_fixture_init() {
+ wrc_require_networkpolicy_lease || return 1
+ command -v jq >/dev/null || return 1
+ E2E_RUN_ID="${E2E_RUN_ID:-$(openssl rand -hex 6)}"
+ # Reject lossy normalization: two supplied IDs must never alias after trimming.
+ [[ "$E2E_RUN_ID" =~ ^[a-z0-9]{1,12}$ ]] || {
+ echo 'E2E_RUN_ID must contain 1–12 lowercase letters or digits' >&2; return 1;
+ }
+ export E2E_RUN_ID
+ local root
+ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
+ mkdir -p "$root/.local-notes/infra/runs"
+ WRC_FIXTURE_DIR="$(mktemp -d "$root/.local-notes/infra/runs/wrc-${E2E_RUN_ID}-XXXXXX")"
+ chmod 700 "$WRC_FIXTURE_DIR"
+ WRC_FIXTURE_LEDGER="$WRC_FIXTURE_DIR/owned.jsonl"
+ (umask 077; : > "$WRC_FIXTURE_LEDGER")
+ jq -n --arg run "$E2E_RUN_ID" --arg profile "$MINIKUBE_PROFILE" \
+ --arg context "$E2E_KUBECONTEXT" --arg worktree "$root" \
+ '{run:$run,profile:$profile,context:$context,worktree:$worktree}' > "$WRC_FIXTURE_DIR/binding.json"
+ WRC_PORT_FORWARD_PID=""
+ log "WRC fixtures E2E_RUN_ID=${E2E_RUN_ID}; evidence=${WRC_FIXTURE_DIR}"
+}
+
+wrc_record_owned() {
+ jq -ce '{apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace,
+ uid:.metadata.uid,labels:(.metadata.labels // {})}} |
+ select(.metadata.uid != null and .metadata.uid != "")' >> "$WRC_FIXTURE_LEDGER"
+}
+
+wrc_create_owned() {
+ local created
+ # create, never apply: a collision cannot change or enroll another run's object.
+ created="$(jq --arg run "$E2E_RUN_ID" '.metadata.labels["e2e.clerum.io/run"]=$run' |
+ kctl create --request-timeout=30s -f - -o json)" || return 1
+ printf '%s\n' "$created" | wrc_record_owned
+}
+
+wrc_delete_owned() {
+ local ns=$1 kind=$2 name=$3 wait_for_delete=${4:-true} owned current uid api_path
+ [[ "$wait_for_delete" == true || "$wait_for_delete" == false ]] || return 1
+ [[ "$ns" =~ ^[a-z0-9][a-z0-9.-]*$ && "$name" =~ ^[a-z0-9][a-z0-9.-]*$ ]] || return 1
+ owned="$(jq -sc --arg ns "$ns" --arg kind "$kind" --arg name "$name" '
+ [.[] | select(.metadata.namespace==$ns and (.kind|ascii_downcase)==($kind|ascii_downcase)
+ and .metadata.name==$name)] | last // empty' "$WRC_FIXTURE_LEDGER")" || return 1
+ [[ -n "$owned" ]] || { echo "Refusing unowned delete: $ns/$kind/$name" >&2; return 1; }
+ current="$(kctl get "$kind" "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1
+ [[ -n "$current" ]] || return 0
+ uid="$(jq -r '.metadata.uid' <<< "$owned")"
+ # UID is the immutable ownership identity. Additional controller labels are
+ # allowed, but every label recorded at enrollment must still match.
+ jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and
+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$current" >/dev/null || {
+ echo "Ownership changed: $ns/$kind/$name" >&2; return 1;
+ }
+ # Use the native DELETE body to enforce UID atomically at the API server;
+ # GET followed by a name-only delete could race Kubernetes GC/recreation.
+ case "$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')" in
+ workflowrecipe) api_path="/apis/clerum.io/v1alpha1/namespaces/$ns/workflowrecipes/$name" ;;
+ networkpolicy) api_path="/apis/networking.k8s.io/v1/namespaces/$ns/networkpolicies/$name" ;;
+ deployment) api_path="/apis/apps/v1/namespaces/$ns/deployments/$name" ;;
+ pod|service|configmap|secret) api_path="/api/v1/namespaces/$ns/$(printf '%s' "$kind" | tr '[:upper:]' '[:lower:]')s/$name" ;;
+ *) echo "Unsupported owned resource kind: $kind" >&2; return 1 ;;
+ esac
+ jq -n --arg uid "$uid" '{apiVersion:"v1",kind:"DeleteOptions",
+ preconditions:{uid:$uid},propagationPolicy:"Background"}' > "$WRC_FIXTURE_DIR/delete-options.json" || return 1
+ kctl delete --raw "$api_path" --request-timeout=30s -f "$WRC_FIXTURE_DIR/delete-options.json" >/dev/null || return 1
+ [[ "$wait_for_delete" == true ]] || return 0
+ kctl wait --for=delete "$kind/$name" -n "$ns" --request-timeout=30s --timeout="${TIMEOUT_DELETE:-90}s" >/dev/null || return 1
+ current="$(kctl get "$kind" "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o name)" || return 1
+ [[ -z "$current" ]]
+}
+
+wrc_capture_recipe_children() {
+ local ns=$1 name=$2 uid=$3 target objects
+ for target in "$ns" sandbox-ui; do
+ objects="$(kctl get deployment,service,configmap,networkpolicy -n "$target" --request-timeout=30s -o json)" || return 1
+ # Cross-namespace UI resources cannot have recipe ownerReferences. WRC's
+ # complete recipe namespace/name labels bind those to our collision-free CR.
+ jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '.items[] |
+ select(any(.metadata.ownerReferences[]?; .uid==$uid) or
+ (.metadata.labels["clerum.io/recipe-namespace"]==$ns and
+ .metadata.labels["clerum.io/recipe-name"]==$name) or
+ (.metadata.namespace==$ns and
+ .metadata.labels["clerum.io/managed-by"]=="workflow-recipes" and
+ .metadata.labels["clerum.io/recipe"]==$name))' <<< "$objects" |
+ while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done || return 1
+ done
+}
+
+wrc_cleanup_owned() {
+ local status=0 ns kind name uid
+ # Capture generated resources before deleting recipes, including the UI
+ # namespace where Kubernetes cannot garbage-collect a cross-namespace owner.
+ while IFS=$'\t' read -r ns name uid; do
+ [[ -n "$name" ]] || continue
+ wrc_capture_recipe_children "$ns" "$name" "$uid" || status=1
+ done < <(jq -r 'select(.kind=="WorkflowRecipe") | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' "$WRC_FIXTURE_LEDGER")
+ # Recipes first stops reconciliation from recreating children during cleanup.
+ while IFS=$'\t' read -r ns kind name; do
+ [[ -n "$name" ]] || continue
+ wrc_delete_owned "$ns" "$kind" "$name" || status=1
+ done < <(jq -sr 'unique_by([.kind,.metadata.namespace,.metadata.name]) |
+ sort_by(if .kind=="WorkflowRecipe" then 0 else 1 end)[] |
+ [.metadata.namespace,.kind,.metadata.name]|@tsv' "$WRC_FIXTURE_LEDGER")
+ return "$status"
+}
+
+wrc_assert_port_forward() {
+ local actual_start command
+ [[ -n "${WRC_PORT_FORWARD_PID:-}" ]] || return 1
+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == live ]] || return 1
+ actual_start="$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" || return 1
+ [[ "$actual_start" == "$WRC_PORT_FORWARD_START" ]] || return 1
+ command="$(pf_owner_process_command "$WRC_PORT_FORWARD_PID")" || return 1
+ [[ "$command" == "$WRC_PORT_FORWARD_COMMAND" ]] || return 1
+ [[ "$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" == "$WRC_PORT_FORWARD_START" ]]
+}
+
+wrc_port_forward_reported_port() {
+ local remote=$1 port
+ pf_owner_validate_port "$remote" || return 1
+ port="$(sed -nE "s/^Forwarding from 127\\.0\\.0\\.1:([0-9]+) -> ${remote}$/\\1/p" "$WRC_FIXTURE_DIR/port-forward.log")" || return 1
+ # Reject absent, duplicated, IPv6-only and out-of-range bindings. The local
+ # port comes exclusively from this child's output, never from a shared URL.
+ pf_owner_validate_port "$port" || return 1
+ printf '%s\n' "$port"
+}
+
+wrc_start_port_forward() {
+ local ns=$1 service=$2 remote=$3 deadline=$((SECONDS + 30)) command
+ [[ -z "${WRC_PORT_FORWARD_PID:-}" && -n "${E2E_KUBECONTEXT:-}" && -n "${MINIKUBE_PROFILE:-}" ]] || return 1
+ pf_owner_validate_port "$remote" || return 1
+ WRC_PORT_FORWARD_PORT=""
+ WRC_PORT_FORWARD_COMMAND="${KUBECTL_BIN} --context=${E2E_KUBECONTEXT} -n ${ns} port-forward --address=127.0.0.1 svc/${service} :${remote}"
+ "$KUBECTL_BIN" "--context=${E2E_KUBECONTEXT}" -n "$ns" port-forward \
+ --address=127.0.0.1 "svc/${service}" ":${remote}" >"$WRC_FIXTURE_DIR/port-forward.log" 2>&1 &
+ WRC_PORT_FORWARD_PID=$!
+ WRC_PORT_FORWARD_START="$(pf_owner_process_start "$WRC_PORT_FORWARD_PID")" || return 1
+ # Capture the exact binary path returned by ps, whose first token may be an
+ # absolute path even when KUBECTL_BIN was resolved through PATH.
+ while (( SECONDS < deadline )); do
+ command="$(pf_owner_process_command "$WRC_PORT_FORWARD_PID")" || return 1
+ if [[ "${command#* }" == "${WRC_PORT_FORWARD_COMMAND#* }" && "${command%% *}" == */kubectl || "$command" == "$WRC_PORT_FORWARD_COMMAND" ]]; then
+ WRC_PORT_FORWARD_COMMAND="$command"
+ if WRC_PORT_FORWARD_PORT="$(wrc_port_forward_reported_port "$remote")"; then
+ pf_owner_write_record_atomic "$WRC_FIXTURE_DIR/port-forward.pid" "$WRC_PORT_FORWARD_PID" \
+ "$WRC_PORT_FORWARD_START" "$MINIKUBE_PROFILE" "$E2E_KUBECONTEXT" "$(pwd -P)" \
+ "$ns" "$service" "$WRC_PORT_FORWARD_PORT" "$remote" || return 1
+ wrc_assert_port_forward
+ return
+ fi
+ fi
+ sleep 0.1
+ done
+ echo 'Owned port-forward did not report a dynamic local binding' >&2
+ return 1
+}
+
+wrc_reap_port_forward() {
+ local record="$WRC_FIXTURE_DIR/port-forward.pid"
+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]] || return 1
+ # Startup can fail before publishing a record. If one exists, preserve it
+ # unless both immutable process identity fields match this exact child.
+ if [[ -e "$record" || -L "$record" ]]; then
+ pf_owner_read_record "$record" || return 1
+ [[ "$PF_OWNER_RECORD_PID" == "$WRC_PORT_FORWARD_PID" &&
+ "$PF_OWNER_RECORD_START" == "$WRC_PORT_FORWARD_START" ]] || return 1
+ fi
+ pf_owner_reap_process "$WRC_PORT_FORWARD_PID"
+ [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]] || return 1
+ pf_owner_remove_dead_record "$record" || return 1
+ WRC_PORT_FORWARD_PID=""
+}
+
+wrc_stop_port_forward() {
+ [[ -n "${WRC_PORT_FORWARD_PID:-}" ]] || return 0
+ local index
+ # A dead/reused process is a conflict, including during successful cleanup.
+ wrc_assert_port_forward || { echo 'Port-forward ownership/liveness conflict' >&2; return 1; }
+ pf_owner_signal_process "$WRC_PORT_FORWARD_PID" TERM || return 1
+ for ((index=0; index<40; index++)); do
+ if [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]]; then
+ wrc_reap_port_forward
+ return
+ fi
+ if ! wrc_assert_port_forward; then
+ # The child can exit between the state probe and ps, after our TERM.
+ if [[ "$(pf_owner_process_state "$WRC_PORT_FORWARD_PID")" == dead ]]; then
+ wrc_reap_port_forward
+ return
+ fi
+ return 1
+ fi
+ sleep 0.1
+ done
+ echo 'Owned port-forward did not exit within bounded cleanup' >&2
+ return 1
+}
diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
index 9cc66bd83..1d9a78336 100755
--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
@@ -4,7 +4,72 @@
set -euo pipefail
-WRC_NP_LAST_RECONCILE_SINCE_TIME=""
+WRC_NP_RECONCILE_BINDING='{}'
+WRC_NP_OBSERVATION_SINCE=""
+WRC_NP_OBSERVATION_PODS='[]'
+WRC_NP_TRACKED='[]'
+
+wrc_require_networkpolicy_lease() {
+ local root
+ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
+ if [ -z "${KUBECONTEXT:-}" ] || [ "$KUBECONTEXT" != "${MINIKUBE_PROFILE:-}" ] ||
+ [ "$KUBECONTEXT" != "${T2_CONTEXT:-${CONTROL_API_REAL_PG_CONTEXT:-}}" ]; then
+ printf 'PROFILE_LOCK_REQUIRED: WRC NetworkPolicy E2E needs the matching explicit owned profile/context\n' >&2
+ return 1
+ fi
+ bash "$root/scripts/minikube/require-t2-mutation-lock.sh"
+}
+
+wrc_now() {
+ python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'
+}
+
+# Track every controller replica and container incarnation, so rolling/restarted
+# observers cannot silently lose the beginning of a no-write window.
+wrc_controller_snapshot() {
+ local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
+ local deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes} definition selector replicas
+ definition="$(kctl get deployment "$deployment" -n "$namespace" -o json)" || return 1
+ selector="$(printf '%s' "$definition" | jq -er '
+ .spec.selector | select((.matchExpressions // [] | length) == 0)
+ | .matchLabels | select(length > 0) | to_entries | sort_by(.key)
+ | map(.key + "=" + .value) | join(",")')" || return 1
+ replicas="$(printf '%s' "$definition" | jq -er '.spec.replicas // 1 | select(. > 0)')" || return 1
+ kctl get pods -n "$namespace" -l "$selector" -o json | jq -ce --argjson replicas "$replicas" '
+ .items | select(length == $replicas)
+ | select(all(.[]; .metadata.deletionTimestamp == null and .status.phase == "Running"
+ and (.status.containerStatuses | length > 0)
+ and all(.status.containerStatuses[]; .ready == true and (.containerID | length > 0))))
+ | map({name:.metadata.name,uid:.metadata.uid,
+ containers:(.status.containerStatuses | sort_by(.name) | map({name,containerID,restartCount}))})
+ | sort_by(.name)'
+}
+
+wrc_controller_logs() {
+ local snapshot=$1 since=$2 pod logs
+ local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
+ while IFS= read -r pod; do
+ [ -n "$pod" ] || return 1
+ logs="$(kctl logs "$pod" -n "$namespace" --all-containers=true --timestamps=true \
+ --since-time="$since")" || return 1
+ printf '%s\n' "$logs"
+ done < <(printf '%s' "$snapshot" | jq -r '.[].name')
+}
+
+# Kubernetes timestamps delimit the cursor; mixed legacy console lines are not
+# JSON events. Rejecting writes is done over all events, never the first match.
+wrc_log_events() {
+ local since=$1
+ jq -Rsc --arg since "$since" '
+ def timestamp_key:
+ sub("Z$"; "") | split(".")
+ | .[0] + "." + (((.[1] // "") + "000000000")[0:9]);
+ ($since | timestamp_key) as $start
+ | split("\n") | map(
+ try (capture("^(?<at>[0-9T:.+-]+Z) (?<payload>.*)$")
+ | select((.at | timestamp_key) >= $start)
+ | .payload | fromjson | select(type == "object")) catch empty)'
+}
wrc_np_spec_hash() {
local namespace=$1 name=$2
@@ -80,14 +145,12 @@ wrc_inject_selector_drift() {
wrc_trigger_recipe_reconcile() {
local namespace=$1 name=$2 timeout=${3:-120}
- local before_generation after_generation marker patch since_time elapsed=0
- local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
- local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}
- local needle="[WR-Reconciler] Reconciling \"${name}\""
+ local before_generation after_generation marker patch since_time started=$SECONDS
+ local recipe uid snapshot events logs
- before_generation="$(
- kctl get workflowrecipe "$name" -n "$namespace" -o jsonpath='{.metadata.generation}'
- )"
+ recipe="$(kctl get workflowrecipe "$name" -n "$namespace" -o json)" || return 1
+ before_generation="$(printf '%s' "$recipe" | jq -er '.metadata.generation')" || return 1
+ uid="$(printf '%s' "$recipe" | jq -er '.metadata.uid | select(length > 0)')" || return 1
[[ "$before_generation" =~ ^[0-9]+$ ]] || {
fail "WorkflowRecipe ${namespace}/${name} has no numeric generation before reconcile trigger"
return 1
@@ -97,10 +160,8 @@ wrc_trigger_recipe_reconcile() {
WRC_NP_RECONCILE_SEQUENCE=$((WRC_NP_RECONCILE_SEQUENCE + 1))
marker="np-e2e-${WRC_NP_RECONCILE_SEQUENCE}-$(date +%s)"
patch="$(jq -cn --arg description "$marker" '{spec:{description:$description}}')"
- since_time="$(
- python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'
- )"
- WRC_NP_LAST_RECONCILE_SINCE_TIME="$since_time"
+ snapshot="$(wrc_controller_snapshot)" || return 1
+ since_time="$(wrc_now)"
kctl patch workflowrecipe "$name" -n "$namespace" --type=merge -p "$patch" >/dev/null
after_generation="$(
@@ -110,80 +171,213 @@ wrc_trigger_recipe_reconcile() {
fail "WorkflowRecipe ${namespace}/${name} generation did not advance (${before_generation} -> ${after_generation})"
return 1
fi
+ WRC_NP_RECONCILE_BINDING="$(jq -cn --arg recipe "$name" --arg namespace "$namespace" \
+ --arg uid "$uid" --argjson generation "$after_generation" '{recipe:$recipe,namespace:$namespace,uid:$uid,generation:$generation}')"
- while [ "$elapsed" -lt "$timeout" ]; do
- if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \
- --since-time="$since_time" 2>/dev/null | grep -Fq "$needle"; then
- ok "WorkflowRecipe ${namespace}/${name} entered reconcile at generation ${after_generation}"
+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do
+ [ "$(wrc_controller_snapshot)" = "$snapshot" ] || {
+ fail "WRC observer changed during the parent reconcile"
+ return 1
+ }
+ logs="$(wrc_controller_logs "$snapshot" "$since_time")" || return 1
+ events="$(printf '%s' "$logs" | wrc_log_events "$since_time")" || return 1
+ if printf '%s' "$events" | jq -e --argjson binding "$WRC_NP_RECONCILE_BINDING" '
+ any(.[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe
+ and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation)' >/dev/null; then
+ ok "WorkflowRecipe ${namespace}/${name} completed reconcile at generation ${after_generation}"
return 0
fi
sleep "$POLL_INTERVAL"
- elapsed=$((elapsed + POLL_INTERVAL))
done
fail "Timed out waiting for WRC to reconcile WorkflowRecipe ${namespace}/${name} generation ${after_generation}"
return 1
}
-wrc_wait_for_np_noop_witness() {
- local namespace=$1 name=$2 family=$3 mode=${4:-apply} timeout=${5:-120} elapsed=0
- local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
- local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}
- local message='network policy unchanged; skipping update'
+wrc_begin_np_observation() {
+ WRC_NP_OBSERVATION_SINCE="$(wrc_now)"
+ WRC_NP_OBSERVATION_PODS="$(wrc_controller_snapshot)" || return 1
+ WRC_NP_TRACKED='[]'
+ WRC_NP_RECONCILE_BINDING='{}'
+}
- [ -n "$WRC_NP_LAST_RECONCILE_SINCE_TIME" ] || {
- fail "No parent reconcile timestamp is available for NetworkPolicy ${namespace}/${name}"
+wrc_track_np() {
+ local namespace=$1 name=$2 family=$3 mode=${4:-apply} policy
+ local message='network policy unchanged; skipping update'
+ [ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_RECONCILE_BINDING" = '{}' ] || {
+ fail 'NetworkPolicy baseline must be captured before the reconcile trigger'
return 1
}
if [ "$mode" = "workload-egress-prefilter" ]; then
message='network policy egress set unchanged; skipping live apply'
elif [ "$mode" != "apply" ]; then
- fail "Unknown NetworkPolicy no-op witness mode: ${mode}"
+ fail "Unknown NetworkPolicy observation mode: ${mode}"
return 1
fi
+ policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json)" || return 1
+ policy="$(printf '%s' "$policy" | jq -ce --arg namespace "$namespace" --arg policy "$name" \
+ --arg family "$family" --arg message "$message" '
+ select(.metadata.deletionTimestamp == null and (.metadata.uid | length > 0)
+ and (.metadata.resourceVersion | length > 0))
+ | {namespace:$namespace,policy:$policy,family:$family,message:$message,
+ uid:.metadata.uid,rv:.metadata.resourceVersion}')" || return 1
+ WRC_NP_TRACKED="$(printf '%s' "$WRC_NP_TRACKED" | jq -c --argjson policy "$policy" '. + [$policy]')"
+}
- while [ "$elapsed" -lt "$timeout" ]; do
- if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \
- --since-time="$WRC_NP_LAST_RECONCILE_SINCE_TIME" 2>/dev/null |
- jq -Rse --arg message "$message" --arg policy "$name" --arg namespace "$namespace" \
- --arg family "$family" '
- split("\n")
- | map(try fromjson catch null)
- | any(
- . != null and
- .msg == $message and
- .policy == $policy and
- .namespace == $namespace and
- .family == $family
- )
- ' >/dev/null; then
- ok "NetworkPolicy ${namespace}/${name} emitted its ${family} no-op witness"
+wrc_assert_np_observation_clean() {
+ local seconds=${1:-20} timeout=${2:-120} started=$SECONDS logs events row current missing
+ [[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [[ "$timeout" =~ ^[1-9][0-9]*$ ]] &&
+ [ "$seconds" -le 300 ] && [ "$timeout" -le 600 ] && [ "$timeout" -ge "$seconds" ] || return 1
+ [ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_TRACKED" != '[]' ] &&
+ [ "$WRC_NP_RECONCILE_BINDING" != '{}' ] || {
+ fail 'NetworkPolicy observation has no baseline, tracked policies, or completed trigger'
+ return 1
+ }
+ while [ "$((SECONDS - started))" -le "$timeout" ]; do
+ [ "$(wrc_controller_snapshot)" = "$WRC_NP_OBSERVATION_PODS" ] || {
+ fail 'WRC observer changed; the NetworkPolicy window is incomplete'
+ return 1
+ }
+ logs="$(wrc_controller_logs "$WRC_NP_OBSERVATION_PODS" "$WRC_NP_OBSERVATION_SINCE")" || {
+ fail 'Unable to read the complete WRC observation window'
+ return 1
+ }
+ events="$(printf '%s' "$logs" | wrc_log_events "$WRC_NP_OBSERVATION_SINCE")" || return 1
+ # Match writes by namespace/name even if a broken writer mislabels family.
+ if printf '%s' "$events" | jq -e --argjson tracked "$WRC_NP_TRACKED" '
+ any(.[]; . as $event | any($tracked[]; .namespace == $event.namespace and .policy == $event.policy)
+ and ((.msg // "") | test("^network policy (created|replaced|create failed|replace failed|read failed)$")))' >/dev/null; then
+ fail 'NetworkPolicy window contains a write or failed operation, even if a no-op also occurred'
+ return 1
+ fi
+ while IFS= read -r row; do
+ current="$(kctl get networkpolicy "$(printf '%s' "$row" | jq -r '.policy')" \
+ -n "$(printf '%s' "$row" | jq -r '.namespace')" -o json)" || return 1
+ if ! printf '%s' "$current" | jq -e --argjson before "$row" '
+ .metadata.uid == $before.uid and .metadata.resourceVersion == $before.rv
+ and .metadata.deletionTimestamp == null' >/dev/null; then
+ fail 'NetworkPolicy changed since its pre-trigger baseline'
+ return 1
+ fi
+ done < <(printf '%s' "$WRC_NP_TRACKED" | jq -c '.[]')
+ missing="$(printf '%s' "$events" | jq -r --argjson tracked "$WRC_NP_TRACKED" \
+ --argjson binding "$WRC_NP_RECONCILE_BINDING" '
+ . as $events | ([ $tracked[] | . as $p | select(any($events[];
+ .policy == $p.policy and .namespace == $p.namespace and .family == $p.family and .msg == $p.message) | not)] | length)
+ + (if any($events[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe
+ and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation
+ and .phase == "active" and .requeueAfterMs == 0) then 0 else 1 end)')" || return 1
+ if [ "$missing" = 0 ] && [ "$((SECONDS - started))" -ge "$seconds" ]; then
+ ok 'Full NetworkPolicy window contains all no-op witnesses, no writes, and unchanged pre-trigger UID/resourceVersion'
+ WRC_NP_OBSERVATION_SINCE=""
return 0
fi
sleep "$POLL_INTERVAL"
- elapsed=$((elapsed + POLL_INTERVAL))
done
-
- fail "NetworkPolicy ${namespace}/${name} emitted no ${family} no-op witness after the parent reconcile"
+ fail 'NetworkPolicy observation lacked an exact completed reconcile or a required no-op witness'
return 1
}
-wrc_assert_np_stable() {
- local namespace=$1 name=$2 settle_seconds=${3:-20}
- local before after
- before="$(wrc_np_resource_version "$namespace" "$name")"
- [ -n "$before" ] || {
- fail "NetworkPolicy ${namespace}/${name} has no resourceVersion before stability window"
- return 1
- }
- # This is an intentional no-churn observation window, not a readiness sleep:
- # the policy is already converged and the assertion is that later reconciles
- # do not update it.
- sleep "$settle_seconds"
- after="$(wrc_np_resource_version "$namespace" "$name")"
- if [ "$after" != "$before" ]; then
- fail "NetworkPolicy ${namespace}/${name} churned while converged (${before} -> ${after})"
+# stdout carries a nonce-bound remote result; stderr/exec failure is a harness
+# error. Only a completed connection timeout is a negative network observation.
+wrc_http_probe() {
+ local namespace=$1 target=$2 host=$3 port=$4 seconds=${E2E_CONNECT_TIMEOUT:-6}
+ local nonce="np-$$-${RANDOM}-${RANDOM}" output begin end outcome encoded rest
+ [[ "$port" =~ ^[0-9]+$ ]] && [ "$port" -gt 0 ] && [ "$port" -le 65535 ] &&
+ [[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [ "$seconds" -le 30 ] || return 1
+ # shellcheck disable=SC2016
+ if ! output="$(kctl exec "$target" -n "$namespace" -- sh -c '
+ export LC_ALL=C
+ for tool in nc grep base64 tr mktemp; do command -v "$tool" >/dev/null 2>&1 || exit 20; done
+ nonce=$1; host=$2; port=$3; seconds=$4
+ errors=$(mktemp) || exit 21
+ trap '\''rm -f "$errors"'\'' EXIT HUP INT TERM
+ printf "WRC_NP_BEGIN|%s\n" "$nonce"
+ response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$seconds" "$host" "$port" 2>"$errors")
+ status=$?
+ if [ "$status" -eq 0 ] && printf "%s" "$response" | grep -Eq "^HTTP/1\.[01] [2345][0-9][0-9]"; then
+ printf "WRC_NP_END|%s|http|" "$nonce"
+ printf "%s" "$response" | base64 | tr -d "\n"
+ printf "\n"
+ elif [ "$status" -ne 0 ] && [ -z "$response" ] && grep -Eqi "timed out|timeout" "$errors"; then
+ printf "WRC_NP_END|%s|timeout|\n" "$nonce"
+ else
+ exit 22
+ fi
+ ' -- "$nonce" "$host" "$port" "$seconds")"; then
+ fail "HTTP probe executor failed for ${namespace}/${target}; this is not NetworkPolicy denial" >&2
return 1
fi
- ok "NetworkPolicy ${namespace}/${name} stayed at resourceVersion ${before}"
+ begin="$(printf '%s\n' "$output" | sed -n '1p')"
+ end="$(printf '%s\n' "$output" | sed -n '2p')"
+ rest="$(printf '%s\n' "$output" | sed -n '3,$p')"
+ [ "$begin" = "WRC_NP_BEGIN|$nonce" ] && [ -z "$rest" ] &&
+ [[ "$end" == "WRC_NP_END|${nonce}|"* ]] || return 1
+ end="${end#WRC_NP_END|${nonce}|}"
+ outcome="${end%%|*}"
+ encoded="${end#*|}"
+ case "$outcome" in
+ http) [ -n "$encoded" ] || return 1 ;;
+ timeout) [ -z "$encoded" ] || return 1 ;;
+ *) return 1 ;;
+ esac
+ jq -cen --arg outcome "$outcome" --arg body "$encoded" '
+ {outcome:$outcome,body:$body}
+ | select((.outcome == "timeout" and .body == "") or
+ (.outcome == "http" and (.body | test("^[A-Za-z0-9+/]+={0,2}$"))
+ and ((.body | length) % 4 == 0)
+ and (.body | @base64d | test("^HTTP/1\\.[01] [2345][0-9][0-9]"))))'
+}
+
+wrc_assert_http_allowed() {
+ local description=$1 namespace=$2 target=$3 host=$4 port=$5 expected=${6:-} result
+ local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}
+ [[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1
+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do
+ result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1
+ if printf '%s' "$result" | jq -e --arg expected "$expected" '
+ .outcome == "http" and ((.body | @base64d) | contains($expected))' >/dev/null; then
+ ok "$description"
+ return 0
+ fi
+ # Policy acknowledgement precedes CNI programming. Only valid remote
+ # network outcomes may be retried; executor failures abort immediately.
+ sleep "$POLL_INTERVAL"
+ done
+ fail "$description: expected an HTTP response from the reachable fixture"
+ return 1
+}
+
+wrc_assert_http_blocked() {
+ local description=$1 namespace=$2 target=$3 host=$4 port=$5 result
+ local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}
+ [[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1
+ while [ "$((SECONDS - started))" -lt "$timeout" ]; do
+ result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1
+ if printf '%s' "$result" | jq -e '.outcome == "timeout"' >/dev/null; then
+ ok "$description"
+ return 0
+ fi
+ sleep "$POLL_INTERVAL"
+ done
+ fail "$description: unexpected successful connection"
+ return 1
+}
+
+# A fixture-owned permission for one direction, peer and port. Complementary
+# controls isolate the boundary being tested without changing platform policy.
+wrc_create_connection_policy() {
+ local name=$1 direction=$2 namespace=$3 selector=$4 peer_namespace=$5 peer_selector=$6 port=$7
+ case "$direction" in Ingress|Egress) ;; *) return 1 ;; esac
+ jq -cn --arg name "$name" --arg direction "$direction" --arg namespace "$namespace" \
+ --argjson selector "$selector" --arg peerNamespace "$peer_namespace" \
+ --argjson peerSelector "$peer_selector" --argjson port "$port" '
+ {namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":$peerNamespace}},
+ podSelector:{matchLabels:$peerSelector}} as $peer
+ | {ports:[{protocol:"TCP",port:$port}]} as $ports
+ | {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",
+ metadata:{name:$name,namespace:$namespace},
+ spec:({podSelector:{matchLabels:$selector},policyTypes:[$direction]} +
+ if $direction == "Ingress" then {ingress:[($ports + {from:[$peer]})]}
+ else {egress:[($ports + {to:[$peer]})]} end)}' | wrc_create_owned
}
diff --git a/scripts/e2e/e2e-sandbox-ui-oauth.sh b/scripts/e2e/e2e-sandbox-ui-oauth.sh
index d0a4cfeea..6f201905a 100755
--- a/scripts/e2e/e2e-sandbox-ui-oauth.sh
+++ b/scripts/e2e/e2e-sandbox-ui-oauth.sh
@@ -31,22 +31,29 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"
+
+if [ "${1:-}" = "--cleanup-only" ]; then
+ echo "Cleanup requires the current run ownership ledger; use normal EXIT cleanup." >&2
+ exit 1
+fi
+wrc_fixture_init
RECIPE_FILE="workflow-recipes/samples/sandbox-ui-oauth-hello.yaml"
-RECIPE_NAME="sandbox-ui-oauth-hello"
+RECIPE_NAME="e2e-oauth-${E2E_RUN_ID}"
WORKLOAD_ID="hello"
SANDBOX_UI_NS="sandbox-ui"
-OAUTH_SECRET_NAME="slack-oauth-creds"
-BACKGROUND_RECIPE_NAME="e2e-wrc-oauth-broker"
+OAUTH_SECRET_NAME="e2e-oauth-secret-${E2E_RUN_ID}"
+BACKGROUND_RECIPE_NAME="e2e-broker-${E2E_RUN_ID}"
BACKGROUND_WORKLOAD_ID="background-worker"
BACKGROUND_POLICY_NAME="wf-${BACKGROUND_RECIPE_NAME}-oauth-broker-egress"
-BACKGROUND_PROBE_POD="e2e-wrc-oauth-unlabelled"
+BACKGROUND_PROBE_POD="e2e-probe-${E2E_RUN_ID}"
STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
TIMEOUT_RECIPE_ACTIVE="${TIMEOUT_RECIPE_ACTIVE:-180}"
UI_DEPLOYMENT=""
BACKGROUND_DEPLOYMENT=""
-UI_PORT_FWD_PORT="${E2E_OAUTH_UI_PORT:-18096}"
-UI_PORT_FWD_PID=""
+# Local UI port is allocated by the owned kubectl child.
CREATED=0
wait_for_recipe_phase() {
@@ -78,21 +85,18 @@ wait_for_workload_instance() {
return 1
}
-http_response_from_target() {
- local target=$1 host=$2 port=$3
- # shellcheck disable=SC2016
- kctl exec "$target" -n "$WORKFLOW_RECIPE_NS" -- sh -c 'printf "GET /health HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w 6 "$1" "$2"' -- "$host" "$port" 2>/dev/null || true
-}
-
create_background_probe() {
- cat <<YAML | kctl apply -f - >/dev/null
+ # This label satisfies Control API's independent ingress policy. The pod
+ # deliberately has no recipe/workload labels selected by OAuth egress.
+ cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: v1
kind: Pod
metadata:
name: ${BACKGROUND_PROBE_POD}
namespace: ${WORKFLOW_RECIPE_NS}
labels:
- e2e.clerum.io/probe: wrc-oauth-unlabelled
+ e2e.clerum.io/probe: ${BACKGROUND_PROBE_POD}
+ clerum.io/oauth-broker-client: "true"
spec:
restartPolicy: Never
securityContext:
@@ -110,50 +114,13 @@ spec:
capabilities:
drop: ["ALL"]
YAML
- wait_for_pod "$WORKFLOW_RECIPE_NS" "e2e.clerum.io/probe=wrc-oauth-unlabelled" 60
+ wait_for_pod "$WORKFLOW_RECIPE_NS" "e2e.clerum.io/probe=${BACKGROUND_PROBE_POD}" 60
}
cleanup() {
local cleanup_status=0
- header "Cleanup"
- if [ -n "$UI_PORT_FWD_PID" ]; then
- if kill -0 "$UI_PORT_FWD_PID" 2>/dev/null; then
- kill "$UI_PORT_FWD_PID" 2>/dev/null || cleanup_status=1
- fi
- wait "$UI_PORT_FWD_PID" 2>/dev/null || true
- UI_PORT_FWD_PID=""
- fi
- kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found \
- --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_status=1
- kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" \
- "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1
- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" \
- "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1
- # WRC GC follows ownerReferences inside the recipe namespace, but UI
- # workloads are reconciled cross-namespace (sandbox-recipes →
- # sandbox-ui) so K8s GC can't follow the link. Force-delete leftovers.
- if [ -n "$UI_DEPLOYMENT" ]; then
- kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- fi
- kctl delete deployment,service -n "$SANDBOX_UI_NS" -l "clerum.io/recipe=${RECIPE_NAME}" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl delete deployment,service,networkpolicy -n "$WORKFLOW_RECIPE_NS" \
- -l "clerum.io/recipe=${BACKGROUND_RECIPE_NAME}" --ignore-not-found \
- >/dev/null 2>&1 || cleanup_status=1
- kctl delete networkpolicy "ui-egress-${RECIPE_NAME}" -n "$SANDBOX_UI_NS" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl delete secret "$OAUTH_SECRET_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
-
+ wrc_stop_port_forward || cleanup_status=1
+ wrc_cleanup_owned || cleanup_status=1
return "$cleanup_status"
}
@@ -161,6 +128,7 @@ on_exit() {
local status=$? cleanup_status=0
trap - EXIT INT TERM
if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then
+ wrc_stop_port_forward || status=1
warn "E2E_KEEP_RESOURCES=1; preserving OAuth fixtures for inspection"
exit "$status"
fi
@@ -178,10 +146,7 @@ on_exit() {
exit "$status"
}
-if [[ "${1:-}" == "--cleanup-only" ]]; then
- cleanup
- exit $?
-fi
+# Resources are cleaned only through this process's ownership ledger.
trap on_exit EXIT
trap 'exit 130' INT
@@ -191,8 +156,7 @@ trap 'exit 143' TERM
check_prerequisites
# ─── Phase 1: Clean Slate ────────────────────────────────────────────
-header "Phase 1 — Clean Slate"
-cleanup >/dev/null 2>&1 || true
+header "Phase 1 — Isolated run ${E2E_RUN_ID}"
CREATED=1
# ─── Phase 2: Create dummy OAuth credentials Secret ──────────────────
@@ -201,12 +165,17 @@ kctl create secret generic "$OAUTH_SECRET_NAME" \
-n "$WORKFLOW_RECIPE_NS" \
--from-literal=client-id="dummy-client-id-for-platform-test" \
--from-literal=client-secret="dummy-client-secret-for-platform-test" \
- >/dev/null
+ --dry-run=client -o json | wrc_create_owned
ok "Secret '${OAUTH_SECRET_NAME}' created in '${WORKFLOW_RECIPE_NS}'"
# ─── Phase 3: Apply Recipe ───────────────────────────────────────────
-apply_recipe "$RECIPE_FILE" "$RECIPE_NAME"
-cat <<YAML | kctl apply -f - >/dev/null
+kctl create --dry-run=client -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" -o json |
+ jq --arg name "$RECIPE_NAME" --arg ns "$WORKFLOW_RECIPE_NS" --arg secret "$OAUTH_SECRET_NAME" '
+ .metadata.name=$name | .metadata.namespace=$ns |
+ .spec.oauthClients[].clientIdRef.name=$secret |
+ .spec.oauthClients[].clientSecretRef.name=$secret' > "$WRC_FIXTURE_DIR/ui-recipe.json"
+wrc_create_owned < "$WRC_FIXTURE_DIR/ui-recipe.json"
+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
@@ -281,36 +250,53 @@ control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.s
exit 1
}
-response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"
-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then
- ok "Opted-in background workload reaches the OAuth broker network boundary"
-else
- fail "Opted-in background workload cannot reach the OAuth broker network boundary"
- exit 1
-fi
+wrc_assert_http_allowed "Opted-in background workload reaches OAuth broker" \
+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090
create_background_probe
-response="$(http_response_from_target "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090)"
-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then
- fail "Unlabelled sandbox workload reached Control API through the OAuth policy"
- exit 1
-fi
-ok "Unlabelled sandbox workload cannot use the OAuth broker egress"
+# Establish reachability from this exact pod before removing the one temporary
+# egress permission. The negative then isolates OAuth egress, not API ingress.
+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: ${BACKGROUND_PROBE_POD}
+ namespace: ${WORKFLOW_RECIPE_NS}
+spec:
+ podSelector:
+ matchLabels:
+ e2e.clerum.io/probe: ${BACKGROUND_PROBE_POD}
+ policyTypes: [Egress]
+ egress:
+ - to:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: ${CONTROL_NS}
+ podSelector:
+ matchLabels:
+ app: control-api
+ ports:
+ - protocol: TCP
+ port: 8090
+YAML
+wrc_assert_http_allowed "OAuth negative probe reaches the same broker with temporary egress" \
+ "$WORKFLOW_RECIPE_NS" "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090
+wrc_delete_owned "$WORKFLOW_RECIPE_NS" networkpolicy "$BACKGROUND_PROBE_POD"
+wrc_assert_http_blocked "Probe without recipe/workload labels cannot use OAuth egress" \
+ "$WORKFLOW_RECIPE_NS" "$BACKGROUND_PROBE_POD" "$control_api_ip" 8090
+wrc_assert_http_allowed "OAuth broker remains reachable after the negative" \
+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090
oauth_policy_hash="$(wrc_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME")"
wrc_inject_selector_drift "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME"
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120
wrc_wait_for_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$oauth_policy_hash" 120
-response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"
-if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then
- ok "OAuth broker route recovered after live NetworkPolicy repair"
-else
- fail "OAuth broker route did not recover after live NetworkPolicy repair"
- exit 1
-fi
+wrc_assert_http_allowed "OAuth broker route recovered after policy repair" \
+ "$WORKFLOW_RECIPE_NS" "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090
+wrc_begin_np_observation
+wrc_track_np "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress apply
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120
-wrc_wait_for_np_noop_witness "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress apply 120
-wrc_assert_np_stable "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$STABILITY_SECONDS"
+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120
# ─── Phase 5: spec.oauthClients[] survives the round-trip ────────────
header "Phase 5 — spec.oauthClients[] preserved on the WorkflowRecipe"
@@ -366,10 +352,8 @@ else
fail "Service '${UI_DEPLOYMENT}' not found in '${SANDBOX_UI_NS}'"
fi
-kctl port-forward -n "$SANDBOX_UI_NS" "service/${UI_DEPLOYMENT}" \
- "${UI_PORT_FWD_PORT}:8080" >/dev/null 2>&1 &
-UI_PORT_FWD_PID=$!
-ui_base_url="http://127.0.0.1:${UI_PORT_FWD_PORT}"
+wrc_start_port_forward "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" 8080
+ui_base_url="http://127.0.0.1:${WRC_PORT_FORWARD_PORT}"
elapsed=0
while [ "$elapsed" -lt 30 ]; do
if curl -fsS --max-time 2 "${ui_base_url}/" >/dev/null 2>&1; then
@@ -378,11 +362,12 @@ while [ "$elapsed" -lt 30 ]; do
sleep "$POLL_INTERVAL"
elapsed=$((elapsed + POLL_INTERVAL))
done
-if ! kill -0 "$UI_PORT_FWD_PID" 2>/dev/null ||
+if ! wrc_assert_port_forward ||
! curl -fsS --max-time 2 "${ui_base_url}/" >/dev/null 2>&1; then
fail "Scoped OAuth UI Service did not become reachable through its port-forward"
exit 1
fi
+wrc_assert_port_forward
ok "Scoped OAuth UI Service serves its visible route"
# ─── Phase 7: UI lifecycle assets serve correctly ────────────────────
diff --git a/scripts/e2e/e2e-webhooks-basic.sh b/scripts/e2e/e2e-webhooks-basic.sh
index bb736c2c9..92902b25c 100755
--- a/scripts/e2e/e2e-webhooks-basic.sh
+++ b/scripts/e2e/e2e-webhooks-basic.sh
@@ -5,7 +5,7 @@
# resources → public POST → verify → forward → handler responds 200.
#
# Phases:
-# 1. Clean slate (delete any leftover from prior runs).
+# 1. Allocate isolated run fixtures and preserve sample resources.
# 2. Create the signing-secret Secret in sandbox-recipes.
# 3. Apply samples/webhook-hello.yaml.
# 4. Wait for the per-recipe webhook-gateway Deployment to be Ready.
@@ -31,20 +31,28 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"
+
+if [ "${1:-}" = "--cleanup-only" ]; then
+ echo "Cleanup requires the current run ownership ledger; use normal EXIT cleanup." >&2
+ exit 1
+fi
+wrc_fixture_init
-require_safe_kube_context
+# The fixture initializer requires the branch mutation lease.
RECIPE_FILE="workflow-recipes/samples/webhook-hello.yaml"
-RECIPE_NAME="webhook-hello"
+RECIPE_NAME="e2e-hook-${E2E_RUN_ID}"
WEBHOOK_ID="hello"
-SECRET_NAME="webhook-hello-creds"
+SECRET_NAME="e2e-hook-secret-${E2E_RUN_ID}"
SECRET_KEY="signing-secret"
SECRET_VALUE="e2e-fireflies-test-secret"
GATEWAY_NAME="wf-${RECIPE_NAME}-webhook-gateway"
GATEWAY_NS="sandbox-recipes"
PROXY_NS="webhook-ingress"
-PROXY_DEPLOYMENT="webhook-proxy"
+PROXY_SERVICE="webhook-proxy"
PROXY_PORT=8095
STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
@@ -52,8 +60,7 @@ PROXY_INGRESS_POLICY="allow-webhook-proxy-ingress-wf-${RECIPE_NAME}"
HANDLER_EGRESS_POLICY="allow-gateway-egress-to-handler-wf-${RECIPE_NAME}"
HANDLER_INGRESS_POLICY="allow-gateway-ingress-to-handler-wf-${RECIPE_NAME}"
-PORT_FWD_PORT=18095
-PORT_FWD_PID=""
+PORT_FWD_PORT=""
HANDLER_DEPLOYMENT=""
CREATED=0
@@ -91,36 +98,8 @@ wait_for_recipe_active() {
cleanup() {
local cleanup_status=0
- if [ -n "$PORT_FWD_PID" ]; then
- if kill -0 "$PORT_FWD_PID" 2>/dev/null; then
- kill "$PORT_FWD_PID" 2>/dev/null || cleanup_status=1
- fi
- wait "$PORT_FWD_PID" 2>/dev/null || true
- PORT_FWD_PID=""
- fi
- kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
- wait_for_workflowrecipe_deleted "$GATEWAY_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \
- >/dev/null 2>&1 || cleanup_status=1
- if [ -n "$HANDLER_DEPLOYMENT" ]; then
- kctl -n "$GATEWAY_NS" delete deployment "$HANDLER_DEPLOYMENT" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete service "$HANDLER_DEPLOYMENT" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- fi
- kctl -n "$GATEWAY_NS" delete deployment,service -l "clerum.io/recipe=${RECIPE_NAME}" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete service "$GATEWAY_NAME" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete configmap "wf-${RECIPE_NAME}-webhook-gateway-config" \
- --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete networkpolicy "$PROXY_INGRESS_POLICY" \
- "$HANDLER_EGRESS_POLICY" "$HANDLER_INGRESS_POLICY" --ignore-not-found \
- >/dev/null 2>&1 || cleanup_status=1
- kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found \
- >/dev/null 2>&1 || cleanup_status=1
+ wrc_stop_port_forward || cleanup_status=1
+ wrc_cleanup_owned || cleanup_status=1
return "$cleanup_status"
}
@@ -128,6 +107,7 @@ on_exit() {
local status=$? cleanup_status=0
trap - EXIT INT TERM
if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then
+ wrc_stop_port_forward || status=1
warn "E2E_KEEP_RESOURCES=1; preserving webhook fixtures for inspection"
exit "$status"
fi
@@ -145,27 +125,20 @@ on_exit() {
exit "$status"
}
-if [ "${1:-}" = "--cleanup-only" ]; then
- cleanup
- exit $?
-fi
+# Resources are cleaned only through this process's ownership ledger.
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# ─── Phase 1: clean slate ───────────────────────────────────────────
-header "Phase 1 — clean slate"
-cleanup >/dev/null 2>&1 || true
+header "Phase 1 — Isolated run ${E2E_RUN_ID}"
CREATED=1
-log "leftover state cleared"
# ─── Phase 2: create signing-secret ─────────────────────────────────
header "Phase 2 — provision signing-secret Secret"
if kctl -n "$GATEWAY_NS" create secret generic "$SECRET_NAME" \
- --from-literal="${SECRET_KEY}=${SECRET_VALUE}" >/dev/null 2>&1; then
- kctl -n "$GATEWAY_NS" label secret "$SECRET_NAME" \
- "clerum.io/owner-recipe=${RECIPE_NAME}" --overwrite >/dev/null
+ --from-literal="${SECRET_KEY}=${SECRET_VALUE}" --dry-run=client -o json | wrc_create_owned; then
ok "Secret $SECRET_NAME/$SECRET_KEY created in $GATEWAY_NS"
else
fail "Secret create failed"
@@ -173,9 +146,13 @@ else
fi
# ─── Phase 3: apply recipe ──────────────────────────────────────────
-header "Phase 3 — apply WorkflowRecipe"
-if kctl apply -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" >/dev/null; then
- ok "WorkflowRecipe applied"
+header "Phase 3 — create isolated WorkflowRecipe"
+kctl create --dry-run=client -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" -o json |
+ jq --arg name "$RECIPE_NAME" --arg ns "$GATEWAY_NS" --arg secret "$SECRET_NAME" '
+ .metadata.name=$name | .metadata.namespace=$ns |
+ .spec.webhooks[].verification.secretRef.name=$secret' > "$WRC_FIXTURE_DIR/webhook-recipe.json"
+if wrc_create_owned < "$WRC_FIXTURE_DIR/webhook-recipe.json"; then
+ ok "Isolated WorkflowRecipe created"
else
fail "WorkflowRecipe apply failed"
exit 1
@@ -233,12 +210,11 @@ fi
# ─── Phase 6: POST a signed payload via webhook-proxy ───────────────
header "Phase 6 — port-forward webhook-proxy and POST signed payload"
-kctl -n "$PROXY_NS" port-forward "deploy/${PROXY_DEPLOYMENT}" "${PORT_FWD_PORT}:${PROXY_PORT}" \
- >/dev/null 2>&1 &
-PORT_FWD_PID=$!
+wrc_start_port_forward "$PROXY_NS" "$PROXY_SERVICE" "$PROXY_PORT"
+PORT_FWD_PORT="$WRC_PORT_FORWARD_PORT"
proxy_ready=0
for _ in $(seq 1 30); do
- kill -0 "$PORT_FWD_PID" 2>/dev/null || break
+ wrc_assert_port_forward || break
if curl -sf --max-time 2 "http://127.0.0.1:${PORT_FWD_PORT}/healthz" >/dev/null; then
proxy_ready=1
break
@@ -246,6 +222,7 @@ for _ in $(seq 1 30); do
sleep 1
done
if [ "$proxy_ready" = "1" ]; then
+ wrc_assert_port_forward
ok "webhook-proxy /healthz reachable via port-forward"
else
fail "webhook-proxy /healthz unreachable"
@@ -265,12 +242,14 @@ SIG="sha256=$(sign "$SECRET_VALUE" "$PAYLOAD")"
http_status() {
local method=$1 path=$2 sig=${3:-} body=${4:-}
- local args=(-s -o /dev/null -w '%{http_code}' -X "$method"
+ wrc_assert_port_forward || return 1
+ local args=(-s --max-time 15 -o "$WRC_FIXTURE_DIR/webhook-response" -w '%{http_code}' -X "$method"
"http://127.0.0.1:${PORT_FWD_PORT}${path}"
-H "content-type: application/json")
[ -n "$sig" ] && args+=(-H "x-hub-signature-256: ${sig}")
[ -n "$body" ] && args+=(--data "$body")
- curl "${args[@]}"
+ curl "${args[@]}" || return 1
+ wrc_assert_port_forward
}
wait_for_policy_owner_uid() {
@@ -288,7 +267,7 @@ wait_for_policy_owner_uid() {
}
status=$(http_status POST "$PUBLIC_PATH" "$SIG" "$PAYLOAD")
-if [ "$status" = "200" ]; then
+if [ "$status" = "200" ] && [ "$(cat "$WRC_FIXTURE_DIR/webhook-response")" = "webhook-ok" ]; then
ok "POST with valid signature → 200"
else
fail "POST with valid signature → ${status} (expected 200)"
@@ -310,7 +289,7 @@ fi
# 7.2 Oversize body via Content-Length. Stream it so the fixture never places a
# multi-megabyte value in argv (which exceeds ARG_MAX on macOS/Linux runners).
-status=$(head -c 2000000 </dev/zero | tr '\0' 'x' | curl -s -o /dev/null -w '%{http_code}' \
+status=$(head -c 2000000 </dev/zero | tr '\0' 'x' | curl -s --max-time 15 -o /dev/null -w '%{http_code}' \
-X POST "http://127.0.0.1:${PORT_FWD_PORT}${PUBLIC_PATH}" \
-H "content-type: application/json" \
-H "content-length: 2000000" \
@@ -374,24 +353,19 @@ wait_for_policy_owner_uid "$HANDLER_INGRESS_POLICY" "$recipe_uid" 120
ok "Webhook gateway policies repaired spec drift, stale owner and missing owner"
status=$(http_status POST "$PUBLIC_PATH" "$SIG" "$PAYLOAD")
-if [ "$status" = "200" ]; then
+if [ "$status" = "200" ] && [ "$(cat "$WRC_FIXTURE_DIR/webhook-response")" = "webhook-ok" ]; then
ok "Signed webhook route recovered after policy repair"
else
fail "Signed webhook route returned ${status} after policy repair"
fi
-proxy_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")"
-handler_egress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")"
-handler_ingress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")"
+wrc_begin_np_observation
+wrc_track_np "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply
+wrc_track_np "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply
+wrc_track_np "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway apply
wrc_trigger_recipe_reconcile "$GATEWAY_NS" "$RECIPE_NAME" 120
-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply 120
-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply 120
-wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway apply 120
-sleep "$STABILITY_SECONDS"
-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")" = "$proxy_rv" ] || fail "${PROXY_INGRESS_POLICY} churned"
-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")" = "$handler_egress_rv" ] || fail "${HANDLER_EGRESS_POLICY} churned"
-[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")" = "$handler_ingress_rv" ] || fail "${HANDLER_INGRESS_POLICY} churned"
-ok "All three webhook gateway policies stayed resourceVersion-stable"
+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120
+ok "All three webhook policies witnessed no-op and stable UID/resourceVersion across the observation"
# ─── Summary ────────────────────────────────────────────────────────
header "Summary"
diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
index b7f186610..0b9f352a5 100755
--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
@@ -9,57 +9,47 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"
-raw_run_id="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
-RUN_ID="$(printf "%s" "$raw_run_id" | tr '[:upper:]' '[:lower:]' \
- | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g' | cut -c1-20)"
-[ -n "$RUN_ID" ] || RUN_ID="run-$$"
-
-RECIPE_NAME="${E2E_RECIPE_NAME:-e2e-wrc-intdep-${RUN_ID}}"
-SOURCE_ID="${E2E_SOURCE_ID:-src-${RUN_ID}}"
-BACKEND_ID="${E2E_BACKEND_ID:-be-${RUN_ID}}"
-DENIED_POD="${E2E_DENIED_POD:-deny-${RUN_ID}}"
+[ "$#" -eq 0 ] || {
+ fail 'This suite accepts no arguments; cleanup is restricted to resources created by this invocation'
+ exit 2
+}
+wrc_fixture_init
+
+RECIPE_NAME="e2e-intdep-${E2E_RUN_ID}"
+SOURCE_ID="src-${E2E_RUN_ID}"
+BACKEND_ID="be-${E2E_RUN_ID}"
+DENIED_POD="deny-${E2E_RUN_ID}"
+PROBE_EGRESS_POLICY="intdep-probe-egress-${E2E_RUN_ID}"
+PROBE_INGRESS_POLICY="intdep-probe-ingress-${E2E_RUN_ID}"
BACKEND_PORT="${E2E_BACKEND_PORT:-8080}"
-CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"
STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
SOURCE_DEPLOYMENT="$SOURCE_ID"
BACKEND_DEPLOYMENT="$BACKEND_ID"
-CREATED=0
+CLEANUP_DONE=0
cleanup() {
- local status=0
- kctl delete pod "$DENIED_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false \
- >/dev/null 2>&1 || status=1
- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \
- >/dev/null 2>&1 || status=1
- kctl delete deployment "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
- --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- kctl delete service "$SOURCE_DEPLOYMENT" "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" \
- --ignore-not-found >/dev/null 2>&1 || status=1
- kctl delete networkpolicy -n "$SANDBOX_NS" \
- -l "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \
- --ignore-not-found >/dev/null 2>&1 || status=1
- return "$status"
+ wrc_cleanup_owned
}
on_exit() {
local status=$?
- if [ "$CREATED" = "1" ] && [ "${E2E_KEEP_RESOURCES:-0}" != "1" ]; then
- cleanup >/dev/null 2>&1 || {
- [ "$status" -ne 0 ] && warn "issue #485 E2E cleanup left resources behind"
- [ "$status" -eq 0 ] && fail "issue #485 E2E cleanup left resources behind" && status=1
- }
+ if [ "$CLEANUP_DONE" -ne 1 ] && ! cleanup; then
+ if [ "$status" -ne 0 ]; then
+ warn 'Internal dependency E2E cleanup also failed; original failure is preserved'
+ else
+ fail 'Internal dependency E2E cleanup failed'
+ status=1
+ fi
fi
exit "$status"
}
trap on_exit EXIT
-if [ "${1:-}" = "--cleanup-only" ]; then
- cleanup
- exit $?
-fi
+[[ "$BACKEND_PORT" =~ ^[0-9]+$ ]] && [ "$BACKEND_PORT" -gt 0 ] &&
+ [ "$BACKEND_PORT" -le 65535 ] || { fail 'Invalid E2E_BACKEND_PORT'; exit 1; }
need_cmd() {
command -v "$1" >/dev/null 2>&1 && ok "Command '$1' available" && return
@@ -174,29 +164,31 @@ log "Context=$(current_e2e_context || true)"
header "Phase 0 - Safety"
need_cmd "$KUBECTL_BIN"
require_safe_kube_context
-kctl cluster-info >/dev/null 2>&1 && ok "Kubernetes cluster reachable" || {
+if ! kctl cluster-info >/dev/null 2>&1; then
fail "Kubernetes cluster not reachable"
exit 1
-}
+fi
+ok "Kubernetes cluster reachable"
for ns in "$WORKFLOW_RECIPE_NS" "$SANDBOX_NS" "$CONTROL_NS"; do
- kctl get ns "$ns" >/dev/null 2>&1 && ok "Namespace ${ns} exists" || {
+ if ! kctl get ns "$ns" >/dev/null 2>&1; then
fail "Namespace ${ns} not found"
exit 1
- }
+ fi
+ ok "Namespace ${ns} exists"
done
-kctl get crd workflowrecipes.clerum.io >/dev/null 2>&1 && ok "WorkflowRecipe CRD installed" || {
+if ! kctl get crd workflowrecipes.clerum.io >/dev/null 2>&1; then
fail "WorkflowRecipe CRD not installed"
exit 1
-}
-kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/null 2>&1 \
- && ok "workflow-recipes rolled out" || {
+fi
+ok "WorkflowRecipe CRD installed"
+if ! kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/null 2>&1; then
fail "workflow-recipes deployment is not ready"
exit 1
- }
+fi
+ok "workflow-recipes rolled out"
-header "Phase 1 - Apply isolated fixture"
-cleanup >/dev/null 2>&1 || true
-cat <<YAML | kctl apply -f -
+header "Phase 1 - Create isolated fixture"
+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
@@ -224,12 +216,11 @@ spec:
args:
- "mkdir -p /www && printf 'issue485-ok\n' > /www/index.html && exec httpd -f -p ${BACKEND_PORT} -h /www"
YAML
-CREATED=1
-ok "WorkflowRecipe fixture applied"
+ok "Run-owned WorkflowRecipe fixture created"
-if kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- -o jsonpath='{.spec.workloads[*].egressBindings}' | grep -q .; then
- fail "Fixture contains egressBindings; that would hide the behavior under test"
+if ! kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o json \
+ | jq -e 'all(.spec.workloads[]; ((.egressBindings // []) | length) == 0)' >/dev/null; then
+ fail "Could not verify the live fixture has no egressBindings shortcut"
exit 1
fi
ok "Fixture has no egressBindings shortcut"
@@ -243,22 +234,25 @@ BACKEND_DEPLOYMENT="$(wait_for_workload_instance "$BACKEND_ID" "$TIMEOUT_POD")"
fail "Backend workload instance was not assigned"
exit 1
}
-wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD" && ok "Source deployment ready" || {
+if ! wait_for_deployment "$SANDBOX_NS" "$SOURCE_DEPLOYMENT" "$TIMEOUT_POD"; then
fail "Source deployment not ready"
exit 1
-}
-wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD" && ok "Backend deployment ready" || {
+fi
+ok "Source deployment ready"
+if ! wait_for_deployment "$SANDBOX_NS" "$BACKEND_DEPLOYMENT" "$TIMEOUT_POD"; then
fail "Backend deployment not ready"
exit 1
-}
+fi
+ok "Backend deployment ready"
wait_internal_ready "$TIMEOUT_POD"
resolved_target="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- printenv TARGET_URL 2>/dev/null || true)"
expected_target="http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/"
-[ "$resolved_target" = "$expected_target" ] && ok "TARGET_URL resolved to ${resolved_target}" || {
+if [ "$resolved_target" != "$expected_target" ]; then
fail "TARGET_URL resolved to '${resolved_target}', expected '${expected_target}'"
exit 1
-}
+fi
+ok "TARGET_URL resolved to ${resolved_target}"
header "Phase 3 - wr-intdep policy shape"
egress_selector="clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME},clerum.io/source-workload=${SOURCE_ID}"
@@ -269,14 +263,9 @@ assert_policy "$egress_ref" "egress" "$SOURCE_ID" "$BACKEND_ID"
assert_policy "$ingress_ref" "ingress" "$BACKEND_ID" "$SOURCE_ID"
header "Phase 4 - Positive packet flow"
-# shellcheck disable=SC2016
-positive_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \
- sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"
-printf "%s" "$positive_output" | grep -Fq "issue485-ok" && \
- ok "WRC source reached backend through inferred internal dependency" || {
- fail "WRC source could not reach backend (output: ${positive_output})"
- exit 1
- }
+wrc_assert_http_allowed 'WRC source reached backend through inferred internal dependency' \
+ "$SANDBOX_NS" "deploy/${SOURCE_DEPLOYMENT}" \
+ "${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local" "$BACKEND_PORT" issue485-ok
header "Phase 5 - Live drift repair and steady no-churn"
egress_ns="${egress_ref%%/*}"
@@ -292,41 +281,25 @@ wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
wrc_wait_for_np_spec_hash "$egress_ns" "$egress_name" "$egress_hash" 120
wrc_wait_for_np_spec_hash "$ingress_ns" "$ingress_name" "$ingress_hash" 120
-# shellcheck disable=SC2016
-repaired_output="$(kctl exec "deploy/${SOURCE_DEPLOYMENT}" -n "$SANDBOX_NS" -- \
- sh -c 'wget -qO- --timeout='"$CONNECT_TIMEOUT"' --tries=1 "$TARGET_URL"' 2>/dev/null || true)"
-printf "%s" "$repaired_output" | grep -Fq "issue485-ok" && \
- ok "Internal dependency traffic recovered after live policy repair" || {
- fail "Internal dependency traffic did not recover after policy repair"
- exit 1
- }
+wrc_assert_http_allowed 'Internal dependency traffic recovered after live policy repair' \
+ "$SANDBOX_NS" "deploy/${SOURCE_DEPLOYMENT}" \
+ "${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local" "$BACKEND_PORT" issue485-ok
-egress_rv="$(wrc_np_resource_version "$egress_ns" "$egress_name")"
-ingress_rv="$(wrc_np_resource_version "$ingress_ns" "$ingress_name")"
+wrc_begin_np_observation
+wrc_track_np "$egress_ns" "$egress_name" internal-dependency
+wrc_track_np "$ingress_ns" "$ingress_name" internal-dependency
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
-wrc_wait_for_np_noop_witness "$egress_ns" "$egress_name" internal-dependency apply 120
-wrc_wait_for_np_noop_witness "$ingress_ns" "$ingress_name" internal-dependency apply 120
-sleep "$STABILITY_SECONDS"
-[ "$(wrc_np_resource_version "$egress_ns" "$egress_name")" = "$egress_rv" ] || {
- fail "${egress_ref} churned after convergence"
- exit 1
-}
-[ "$(wrc_np_resource_version "$ingress_ns" "$ingress_name")" = "$ingress_rv" ] || {
- fail "${ingress_ref} churned after convergence"
- exit 1
-}
-ok "Both internal-dependency policies stayed resourceVersion-stable"
+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120
header "Phase 6 - Negative packet flow"
-kctl delete pod "$DENIED_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true
-cat <<YAML | kctl apply -f - >/dev/null
+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: v1
kind: Pod
metadata:
name: ${DENIED_POD}
namespace: ${SANDBOX_NS}
labels:
- run: ${DENIED_POD}
+ e2e.clerum.io/probe: ${DENIED_POD}
e2e.clerum.io/suite: wrc-internal-dependency-networkpolicy
spec:
restartPolicy: Never
@@ -345,24 +318,40 @@ spec:
capabilities:
drop: ["ALL"]
YAML
-wait_for_pod "$SANDBOX_NS" "run=${DENIED_POD}" 60 && ok "Unlabeled negative pod ready" || {
- fail "Unlabeled negative pod not ready"
- exit 1
-}
-if kctl exec "$DENIED_POD" -n "$SANDBOX_NS" -- wget -qO- \
- --timeout="$CONNECT_TIMEOUT" --tries=1 \
- "http://${BACKEND_DEPLOYMENT}.${SANDBOX_NS}.svc.cluster.local:${BACKEND_PORT}/" >/dev/null 2>&1; then
- fail "Unlabeled pod reached backend; policy is too broad or NetworkPolicy is not enforced"
+if ! wait_for_pod "$SANDBOX_NS" "e2e.clerum.io/probe=${DENIED_POD}" 60; then
+ fail "Rogue probe pod not ready"
exit 1
fi
-ok "Unlabeled pod cannot reach backend"
+ok "Rogue probe pod ready"
+
+# The rogue source must have a working egress path before testing backend
+# ingress. Use the Service IP so denied DNS cannot masquerade as ingress denial.
+backend_ip="$(kctl get service "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"
+[[ "$backend_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
+ fail 'Backend Service has no IPv4 ClusterIP for the isolated ingress control'
+ exit 1
+}
+backend_pod_selector="$(kctl get deployment "$BACKEND_DEPLOYMENT" -n "$SANDBOX_NS" -o json \
+ | jq -ce '.spec.selector.matchLabels | select(type == "object" and length > 0)')"
+probe_pod_selector="$(jq -cn --arg probe "$DENIED_POD" '{"e2e.clerum.io/probe":$probe}')"
+wrc_create_connection_policy "$PROBE_EGRESS_POLICY" Egress \
+ "$SANDBOX_NS" "$probe_pod_selector" "$SANDBOX_NS" "$backend_pod_selector" "$BACKEND_PORT"
+wrc_create_connection_policy "$PROBE_INGRESS_POLICY" Ingress \
+ "$SANDBOX_NS" "$backend_pod_selector" "$SANDBOX_NS" "$probe_pod_selector" "$BACKEND_PORT"
+wrc_assert_http_allowed 'Same rogue source reaches healthy backend with temporary ingress' \
+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT" issue485-ok
+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$PROBE_INGRESS_POLICY"
+wrc_assert_http_blocked 'Internal dependency ingress rejects a source outside its workload selector' \
+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT"
+wrc_create_connection_policy "$PROBE_INGRESS_POLICY" Ingress \
+ "$SANDBOX_NS" "$backend_pod_selector" "$SANDBOX_NS" "$probe_pod_selector" "$BACKEND_PORT"
+wrc_assert_http_allowed 'Same rogue source and backend remain healthy after the negative control' \
+ "$SANDBOX_NS" "$DENIED_POD" "$backend_ip" "$BACKEND_PORT" issue485-ok
+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$PROBE_INGRESS_POLICY"
header "Phase 7 - Cleanup"
cleanup
-CREATED=0
-kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" >/dev/null 2>&1 \
- && fail "WorkflowRecipe still exists after cleanup" || ok "WorkflowRecipe removed"
-policy_refs "clerum.io/managed-by=workflow-recipes,clerum.io/policy-type=internal-dependency,clerum.io/recipe=${RECIPE_NAME}" \
- | grep -q . && fail "wr-intdep policies still exist after cleanup" || ok "wr-intdep policies removed"
+CLEANUP_DONE=1
+ok 'Run-owned fixture resources removed with UID-checked cleanup'
print_results
diff --git a/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh b/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
index 73e8248bf..2224d3201 100755
--- a/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
+++ b/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh
@@ -6,6 +6,9 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${KUBECONTEXT:?KUBECONTEXT is required for WRC NetworkPolicy E2E}"
+# shellcheck source=scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
+source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"
+wrc_require_networkpolicy_lease
suites=(
e2e-wrc-networkpolicy-service-routes.sh
diff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
index 825d36f53..99d8afdc5 100755
--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
@@ -8,61 +8,68 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${SCRIPT_DIR}/e2e-lib.sh"
# shellcheck disable=SC1091
source "${SCRIPT_DIR}/_lib/wrc-networkpolicy-convergence.sh"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/_lib/wrc-fixtures.sh"
+wrc_fixture_init
-raw_run_id="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
-RUN_ID="$(printf '%s' "$raw_run_id" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g; s/^-+//; s/-+$//; s/-+/-/g' | cut -c1-16)"
-[ -n "$RUN_ID" ] || RUN_ID="run-$$"
-
-RECIPE_NAME="${E2E_RECIPE_NAME:-e2e-wrc-routes-${RUN_ID}}"
+RECIPE_NAME="e2e-routes-${E2E_RUN_ID}"
SANDBOX_UI_NS="${SANDBOX_UI_NS:-sandbox-ui}"
UI_ID="ui"
API_ID="api"
DB_ID="db"
+OTHER_DB_ID="other-db"
DB_PORT=8081
-CONNECT_TIMEOUT="${E2E_CONNECT_TIMEOUT:-6}"
STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
ROGUE_UI_POD="${RECIPE_NAME}-rogue-ui"
ROGUE_WORKLOAD_POD="${RECIPE_NAME}-rogue-wl"
CREATED=0
HELD_POLICY_FINALIZER=0
+HELD_POLICY_UID=""
UI_DEPLOYMENT=""
API_DEPLOYMENT=""
DB_DEPLOYMENT=""
+OTHER_DB_DEPLOYMENT=""
UI_INGRESS_POLICY="ui-ingress-${RECIPE_NAME}-${API_ID}"
WL_EGRESS_POLICY="wl-egress-${RECIPE_NAME}-${API_ID}"
WL_INGRESS_POLICY="wl-ingress-${RECIPE_NAME}-${DB_ID}"
cleanup() {
- local status=0 namespace
- kctl delete pod "$ROGUE_UI_POD" -n "$SANDBOX_UI_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- kctl delete pod "$ROGUE_WORKLOAD_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- if [ "$HELD_POLICY_FINALIZER" = "1" ] &&
- kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" >/dev/null 2>&1; then
- kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
- -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || status=1
- HELD_POLICY_FINALIZER=0
- fi
- kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
- wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" >/dev/null 2>&1 || status=1
- for namespace in "$SANDBOX_NS" "$SANDBOX_UI_NS"; do
- kctl delete networkpolicy -n "$namespace" -l "clerum.io/recipe=${RECIPE_NAME}" --ignore-not-found >/dev/null 2>&1 || status=1
- done
+ local status=0
+ if [ "$HELD_POLICY_FINALIZER" = "1" ]; then release_policy_hold || status=1; fi
+ wrc_cleanup_owned || status=1
return "$status"
}
+release_policy_hold() {
+ local policy patch
+ policy="$(kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --ignore-not-found -o json)" || return 1
+ [ -n "$policy" ] || { HELD_POLICY_FINALIZER=0; return 0; }
+ patch="$(printf '%s' "$policy" | jq -ce --arg uid "$HELD_POLICY_UID" '
+ select(.metadata.uid == $uid)
+ | [{op:"test",path:"/metadata/uid",value:$uid},
+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},
+ {op:"replace",path:"/metadata/finalizers",value:[.metadata.finalizers[] | select(. != "e2e.invalid/hold-deletion")]}]')" || return 1
+ kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=json -p "$patch" >/dev/null || return 1
+ HELD_POLICY_FINALIZER=0
+}
+
on_exit() {
- local status=$?
+ local status=$? cleanup_status=0
+ trap - EXIT INT TERM
if [ "$CREATED" = "1" ] && [ "${E2E_KEEP_RESOURCES:-0}" != "1" ]; then
- cleanup >/dev/null 2>&1 || {
- [ "$status" -ne 0 ] && warn "WRC service-route E2E cleanup left resources behind"
- [ "$status" -eq 0 ] && fail "WRC service-route E2E cleanup left resources behind" && status=1
- }
+ cleanup || cleanup_status=$?
+ fi
+ if [ "$cleanup_status" -ne 0 ]; then
+ warn 'WRC service-route E2E cleanup did not complete'
+ if [ "$status" -eq 0 ]; then status=1; fi
fi
exit "$status"
}
trap on_exit EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
wait_for_workload_instance() {
local workload_id=$1 timeout=${2:-$TIMEOUT_POD} elapsed=0 instance
@@ -111,31 +118,9 @@ wait_for_recipe_message() {
return 1
}
-assert_http_body() {
- local description=$1 namespace=$2 deployment=$3 host=$4 port=$5 expected=$6 body
- # shellcheck disable=SC2016
- body="$(kctl exec "deploy/${deployment}" -n "$namespace" -- sh -c 'printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$1" "$2" "$3"' -- "$CONNECT_TIMEOUT" "$host" "$port" 2>/dev/null || true)"
- if printf '%s' "$body" | grep -Fq "$expected"; then
- ok "$description"
- return 0
- fi
- fail "${description} (unexpected response)"
- return 1
-}
-
-assert_pod_blocked() {
- local description=$1 namespace=$2 pod=$3 host=$4 port=$5
- # shellcheck disable=SC2016
- if kctl exec "$pod" -n "$namespace" -- sh -c 'printf "GET / HTTP/1.0\r\n\r\n" | nc -w "$1" "$2" "$3"' -- "$CONNECT_TIMEOUT" "$host" "$port" >/dev/null 2>&1; then
- fail "${description}: unexpected connection succeeded"
- return 1
- fi
- ok "$description"
-}
-
apply_rogue_pod() {
local namespace=$1 name=$2
- cat <<YAML | kctl apply -f - >/dev/null
+ cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: v1
kind: Pod
metadata:
@@ -164,26 +149,6 @@ YAML
wait_for_pod "$namespace" "e2e.clerum.io/probe=${name}" 60
}
-assert_all_stable() {
- local ui_before wl_egress_before wl_ingress_before
- local ui_after wl_egress_after wl_ingress_after
- ui_before="$(wrc_np_resource_version "$SANDBOX_NS" "$UI_INGRESS_POLICY")"
- wl_egress_before="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_EGRESS_POLICY")"
- wl_ingress_before="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_INGRESS_POLICY")"
-
- # Intentional no-churn observation window after all readiness assertions.
- sleep "$STABILITY_SECONDS"
-
- ui_after="$(wrc_np_resource_version "$SANDBOX_NS" "$UI_INGRESS_POLICY")"
- wl_egress_after="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_EGRESS_POLICY")"
- wl_ingress_after="$(wrc_np_resource_version "$SANDBOX_NS" "$WL_INGRESS_POLICY")"
-
- [ "$ui_after" = "$ui_before" ] || fail "${UI_INGRESS_POLICY} churned"
- [ "$wl_egress_after" = "$wl_egress_before" ] || fail "${WL_EGRESS_POLICY} churned"
- [ "$wl_ingress_after" = "$wl_ingress_before" ] || fail "${WL_INGRESS_POLICY} churned"
- ok "All three converged NetworkPolicies stayed resourceVersion-stable"
-}
-
header "WRC NetworkPolicy correctly-configured service routes"
require_safe_kube_context
kctl cluster-info >/dev/null 2>&1 || {
@@ -193,8 +158,8 @@ kctl cluster-info >/dev/null 2>&1 || {
kctl -n "$CONTROL_NS" rollout status deploy/workflow-recipes --timeout=120s >/dev/null
header "Phase 1 — isolated fixture"
-cleanup >/dev/null 2>&1 || true
-cat <<YAML | kctl apply -f - >/dev/null
+CREATED=1
+cat <<YAML | kctl create --dry-run=client -f - -o json | wrc_create_owned
apiVersion: clerum.io/v1alpha1
kind: WorkflowRecipe
metadata:
@@ -230,6 +195,13 @@ spec:
command: ["sh", "-c"]
args:
- "mkdir -p /www && printf 'db-ok\n' > /www/index.html && exec httpd -f -p ${DB_PORT} -h /www"
+ - id: ${OTHER_DB_ID}
+ type: deployment
+ image: busybox:1.36.1
+ port: ${DB_PORT}
+ command: ["sh", "-c"]
+ args:
+ - "mkdir -p /www && printf 'other-db-ok\n' > /www/index.html && exec httpd -f -p ${DB_PORT} -h /www"
ui:
workloadRef: ${UI_ID}
port: 8080
@@ -240,12 +212,12 @@ spec:
- workloadRef: ${API_ID}
port: 8080
YAML
-CREATED=1
wait_for_recipe_active 180
UI_DEPLOYMENT="$(wait_for_workload_instance "$UI_ID" 120)"
API_DEPLOYMENT="$(wait_for_workload_instance "$API_ID" 120)"
DB_DEPLOYMENT="$(wait_for_workload_instance "$DB_ID" 120)"
+OTHER_DB_DEPLOYMENT="$(wait_for_workload_instance "$OTHER_DB_ID" 120)"
[ -n "$UI_DEPLOYMENT" ] && [ -n "$API_DEPLOYMENT" ] && [ -n "$DB_DEPLOYMENT" ] || {
fail "One or more workload instance names are missing"
exit 1
@@ -254,6 +226,7 @@ DB_DEPLOYMENT="$(wait_for_workload_instance "$DB_ID" 120)"
wait_for_deployment "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" 180
wait_for_deployment "$SANDBOX_NS" "$API_DEPLOYMENT" 180
wait_for_deployment "$SANDBOX_NS" "$DB_DEPLOYMENT" 180
+wait_for_deployment "$SANDBOX_NS" "$OTHER_DB_DEPLOYMENT" 180
for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; do
wrc_wait_for_np "$SANDBOX_NS" "$policy" 120
@@ -262,20 +235,58 @@ ok "ui-ingress, workload-egress and workload-ingress materialized"
API_CLUSTER_IP="$(kctl get service "$API_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"
DB_CLUSTER_IP="$(kctl get service "$DB_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"
+OTHER_DB_CLUSTER_IP="$(kctl get service "$OTHER_DB_DEPLOYMENT" -n "$SANDBOX_NS" -o jsonpath='{.spec.clusterIP}')"
[ -n "$API_CLUSTER_IP" ] && [ -n "$DB_CLUSTER_IP" ] || {
fail "Fixture Service ClusterIP discovery failed"
exit 1
}
header "Phase 2 — correctly configured positive routes"
-assert_http_body "sandbox UI reaches declared API backend" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"
-assert_http_body "API workload reaches declared sibling DB" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
+wrc_assert_http_allowed "sandbox UI reaches declared API backend" "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"
+wrc_assert_http_allowed "API workload reaches declared sibling DB" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
header "Phase 3 — negative controls"
apply_rogue_pod "$SANDBOX_UI_NS" "$ROGUE_UI_POD"
apply_rogue_pod "$SANDBOX_NS" "$ROGUE_WORKLOAD_POD"
-assert_pod_blocked "unlabelled sandbox-ui pod cannot reach recipe API" "$SANDBOX_UI_NS" "$ROGUE_UI_POD" "$API_CLUSTER_IP" 8080
-assert_pod_blocked "unlabelled sandbox workload cannot reach recipe DB" "$SANDBOX_NS" "$ROGUE_WORKLOAD_POD" "$DB_CLUSTER_IP" "$DB_PORT"
+api_selector="$(kctl get deployment "$API_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"
+db_selector="$(kctl get deployment "$DB_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"
+other_db_selector="$(kctl get deployment "$OTHER_DB_DEPLOYMENT" -n "$SANDBOX_NS" -o json | jq -ce '.spec.selector.matchLabels')"
+ui_probe_selector="$(jq -cn --arg probe "$ROGUE_UI_POD" '{"e2e.clerum.io/probe":$probe}')"
+wl_probe_selector="$(jq -cn --arg probe "$ROGUE_WORKLOAD_POD" '{"e2e.clerum.io/probe":$probe}')"
+
+# Leave only the ingress under test as the rejecting boundary. The same rogue
+# must first prove it can execute and reach the healthy backend with a narrow
+# temporary ingress permission, and must lose access when that permission goes.
+wrc_create_connection_policy "${RECIPE_NAME}-ui-source" Egress "$SANDBOX_UI_NS" "$ui_probe_selector" "$SANDBOX_NS" "$api_selector" 8080
+wrc_create_connection_policy "${RECIPE_NAME}-wl-source" Egress "$SANDBOX_NS" "$wl_probe_selector" "$SANDBOX_NS" "$db_selector" "$DB_PORT"
+for lane in ui wl; do
+ if [ "$lane" = ui ]; then
+ probe_ns=$SANDBOX_UI_NS; probe=$ROGUE_UI_POD; probe_selector=$ui_probe_selector
+ target_selector=$api_selector; target_ip=$API_CLUSTER_IP; target_port=8080; expected=api-ok
+ else
+ probe_ns=$SANDBOX_NS; probe=$ROGUE_WORKLOAD_POD; probe_selector=$wl_probe_selector
+ target_selector=$db_selector; target_ip=$DB_CLUSTER_IP; target_port=$DB_PORT; expected=db-ok
+ fi
+ control="${RECIPE_NAME}-${lane}-target"
+ wrc_create_connection_policy "$control" Ingress "$SANDBOX_NS" "$target_selector" "$probe_ns" "$probe_selector" "$target_port"
+ wrc_assert_http_allowed "$lane negative probe reaches the healthy target with its control permission" "$probe_ns" "$probe" "$target_ip" "$target_port" "$expected"
+ wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$control"
+ wrc_assert_http_blocked "$lane ingress rejects the rogue when only its target permission is removed" "$probe_ns" "$probe" "$target_ip" "$target_port"
+ wrc_create_connection_policy "$control" Ingress "$SANDBOX_NS" "$target_selector" "$probe_ns" "$probe_selector" "$target_port"
+ wrc_assert_http_allowed "$lane negative control remains healthy after the denial" "$probe_ns" "$probe" "$target_ip" "$target_port" "$expected"
+ wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$control"
+done
+
+# For workload egress, allow only the complementary destination ingress. An
+# undeclared sibling must remain unreachable from the actual API workload.
+wrc_create_connection_policy "${RECIPE_NAME}-other-target" Ingress "$SANDBOX_NS" "$other_db_selector" "$SANDBOX_NS" "$api_selector" "$DB_PORT"
+wrc_create_connection_policy "${RECIPE_NAME}-other-source" Egress "$SANDBOX_NS" "$api_selector" "$SANDBOX_NS" "$other_db_selector" "$DB_PORT"
+wrc_assert_http_allowed 'API probe reaches undeclared sibling with temporary egress permission' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT" 'other-db-ok'
+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "${RECIPE_NAME}-other-source"
+wrc_assert_http_blocked 'Workload egress rejects the undeclared sibling with destination ingress allowed' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT"
+wrc_create_connection_policy "${RECIPE_NAME}-other-source" Egress "$SANDBOX_NS" "$api_selector" "$SANDBOX_NS" "$other_db_selector" "$DB_PORT"
+wrc_assert_http_allowed 'Undeclared sibling remains healthy after egress denial' "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$OTHER_DB_CLUSTER_IP" "$DB_PORT" 'other-db-ok'
+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "${RECIPE_NAME}-other-source"
header "Phase 4 — live drift repair"
ui_hash="$(wrc_np_spec_hash "$SANDBOX_NS" "$UI_INGRESS_POLICY")"
@@ -292,41 +303,47 @@ wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_EGRESS_POLICY" "$wl_egress_hash" 12
wrc_wait_for_np_spec_hash "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$wl_ingress_hash" 120
ok "WRC repaired live spec drift for all three families"
-assert_http_body "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"
-assert_http_body "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
+wrc_assert_http_allowed "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "deploy/$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"
+wrc_assert_http_allowed "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
header "Phase 5 — terminating race self-heals without another parent event"
terminating_uid="$(wrc_np_uid "$SANDBOX_NS" "$WL_INGRESS_POLICY")"
+HELD_POLICY_UID=$terminating_uid
terminating_generation="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')"
-kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
- -p '{"metadata":{"finalizers":["e2e.invalid/hold-deletion"]}}' >/dev/null
+held_policy="$(kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" -o json)"
+printf '%s' "$held_policy" | wrc_record_owned
+hold_patch="$(printf '%s' "$held_policy" | jq -c '[
+ {op:"test",path:"/metadata/uid",value:.metadata.uid},
+ {op:"test",path:"/metadata/resourceVersion",value:.metadata.resourceVersion},
+ {op:"add",path:"/metadata/finalizers",value:((.metadata.finalizers // []) + ["e2e.invalid/hold-deletion"])}]')"
+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=json -p "$hold_patch" >/dev/null
HELD_POLICY_FINALIZER=1
-kctl delete networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --wait=false >/dev/null
+wrc_delete_owned "$SANDBOX_NS" NetworkPolicy "$WL_INGRESS_POLICY" false
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
wait_for_recipe_message 'is terminating; retrying after deletion' 120
-kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
- -p '{"metadata":{"finalizers":[]}}' >/dev/null
-HELD_POLICY_FINALIZER=0
+release_policy_hold
wrc_wait_for_np_recreated "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$terminating_uid" "$wl_ingress_hash" 120
[ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')" = "$((terminating_generation + 1))" ] || {
fail "NetworkPolicy recovery depended on an unexpected second parent spec event"
exit 1
}
wait_for_recipe_active 120
-assert_http_body "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
+wrc_assert_http_allowed "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "deploy/$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
header "Phase 6 — steady-state no-churn"
+wrc_begin_np_observation
+wrc_track_np "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress
+wrc_track_np "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter
+wrc_track_np "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingress
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress apply 120
-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter 120
-wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingress apply 120
-assert_all_stable
+wrc_assert_np_observation_clean "$STABILITY_SECONDS" 120
header "Phase 7 — cleanup"
cleanup
CREATED=0
for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; do
- if kctl get networkpolicy "$policy" -n "$SANDBOX_NS" >/dev/null 2>&1; then
+ remaining="$(kctl get networkpolicy "$policy" -n "$SANDBOX_NS" --ignore-not-found -o name)"
+ if [ -n "$remaining" ]; then
fail "NetworkPolicy ${SANDBOX_NS}/${policy} survived cleanup"
exit 1
fi
diff --git a/scripts/tests/test-wrc-fixtures.sh b/scripts/tests/test-wrc-fixtures.sh
new file mode 100755
index 000000000..7fd15cded
--- /dev/null
+++ b/scripts/tests/test-wrc-fixtures.sh
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+# Hermetic API/process boundary contracts; never invokes kubectl or a cluster.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+# shellcheck disable=SC1091
+source "$ROOT/scripts/e2e/_lib/wrc-fixtures.sh"
+TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-fixtures.XXXXXX")"
+trap 'rm -rf -- "$TEST_ROOT"' EXIT
+WRC_FIXTURE_DIR="$TEST_ROOT"
+WRC_FIXTURE_LEDGER="$TEST_ROOT/owned.jsonl"
+E2E_RUN_ID=contract123
+MODE=normal
+
+reset_fixture() {
+ MODE=normal
+ : > "$WRC_FIXTURE_LEDGER"
+ : > "$TEST_ROOT/calls"
+ rm -f "$TEST_ROOT/live.json"
+}
+
+kctl() {
+ printf '%s\n' "$*" >> "$TEST_ROOT/calls"
+ case "$1" in
+ create)
+ [[ ! -e "$TEST_ROOT/live.json" ]] || return 1
+ jq '.metadata.uid="created-uid"' > "$TEST_ROOT/live.json"
+ cat "$TEST_ROOT/live.json"
+ ;;
+ get)
+ [[ "$MODE" != read-error ]] || return 1
+ if [[ "$2" == deployment,service,configmap,networkpolicy ]]; then
+ cat "$TEST_ROOT/children.json"
+ return
+ fi
+ [[ -e "$TEST_ROOT/live.json" ]] || return 0
+ if [[ " $* " == *' -o name '* ]]; then
+ printf 'configmap/fixture\n'
+ else
+ cat "$TEST_ROOT/live.json"
+ fi
+ ;;
+ delete)
+ [[ "$2" == --raw && "$3" == /api/v1/namespaces/sandbox-recipes/configmaps/fixture ]] || return 1
+ [[ "$4" == --request-timeout=30s && "$5" == -f ]] || return 1
+ jq -e '.kind=="DeleteOptions" and .preconditions.uid=="created-uid"' "$6" >/dev/null || return 1
+ [[ "$MODE" != delete-error ]] || return 1
+ if [[ "$MODE" == recreate-race ]]; then
+ jq '.metadata.uid="replacement-uid"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"
+ mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"
+ fi
+ # Simulate the API server's atomic precondition rather than allowing a
+ # name-only delete to make the ownership-race contract vacuously green.
+ [[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == "$(jq -r .preconditions.uid "$6")" ]] || return 1
+ rm "$TEST_ROOT/live.json"
+ ;;
+ wait) [[ ! -e "$TEST_ROOT/live.json" ]] ;;
+ *) return 1 ;;
+ esac
+}
+
+create_fixture() {
+ jq -n '{apiVersion:"v1",kind:"ConfigMap",metadata:{namespace:"sandbox-recipes",name:"fixture"},
+ data:{message:"not retained in ownership evidence"}}' | wrc_create_owned
+}
+
+must_reject() {
+ if "$@" > "$TEST_ROOT/rejected.out" 2>&1; then
+ echo "Expected rejection: $*" >&2
+ exit 1
+ fi
+}
+
+reset_fixture
+create_fixture
+jq -e '.metadata.uid=="created-uid" and .metadata.labels["e2e.clerum.io/run"]=="contract123" and (has("data")|not)' \
+ "$WRC_FIXTURE_LEDGER" >/dev/null
+must_reject create_fixture
+[[ "$(wc -l < "$WRC_FIXTURE_LEDGER" | tr -d ' ')" == 1 ]]
+[[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == created-uid ]]
+echo 'PASS: collision preserves prior object and does not enroll it twice'
+
+MODE=read-error
+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture
+[[ "$(rg -c '^delete ' "$TEST_ROOT/calls" || true)" == '' ]]
+MODE=normal
+jq '.metadata.uid="replacement-uid"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"
+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"
+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture
+[[ "$(rg -c '^delete ' "$TEST_ROOT/calls" || true)" == '' ]]
+echo 'PASS: read errors and changed UID refuse deletion'
+
+reset_fixture
+create_fixture
+jq '.metadata.labels["e2e.clerum.io/run"]="foreign"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"
+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"
+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture
+[[ -e "$TEST_ROOT/live.json" ]]
+echo 'PASS: changed ownership label refuses deletion'
+
+reset_fixture
+create_fixture
+jq '.metadata.labels["controller-added"]="retained"' "$TEST_ROOT/live.json" > "$TEST_ROOT/replacement.json"
+mv "$TEST_ROOT/replacement.json" "$TEST_ROOT/live.json"
+wrc_delete_owned sandbox-recipes ConfigMap fixture
+[[ ! -e "$TEST_ROOT/live.json" ]]
+echo 'PASS: extra controller labels accepted; DELETE carries exact UID and proves absence'
+
+reset_fixture
+create_fixture
+MODE=recreate-race
+must_reject wrc_delete_owned sandbox-recipes ConfigMap fixture
+[[ "$(jq -r .metadata.uid "$TEST_ROOT/live.json")" == replacement-uid ]]
+echo 'PASS: API UID precondition preserves concurrent replacement'
+
+reset_fixture
+create_fixture
+MODE=delete-error
+must_reject wrc_cleanup_owned
+[[ -e "$TEST_ROOT/live.json" ]]
+echo 'PASS: cleanup failure is nonzero and leaves evidence'
+
+reset_fixture
+# Only owner UID, complete cross-namespace recipe binding, or WRC's exact
+# same-namespace managed recipe label can enroll a generated child.
+jq -n '{items:[
+ {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"ownerless",namespace:"sandbox-recipes",uid:"child1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"fixture"}}},
+ {apiVersion:"v1",kind:"Service",metadata:{name:"cross-ns",namespace:"sandbox-ui",uid:"child2",labels:{"clerum.io/recipe-namespace":"sandbox-recipes","clerum.io/recipe-name":"fixture"}}},
+ {apiVersion:"apps/v1",kind:"Deployment",metadata:{name:"owned",namespace:"sandbox-recipes",uid:"child3",ownerReferences:[{uid:"recipe-uid"}]}},
+ {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"foreign",namespace:"sandbox-recipes",uid:"foreign1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"another"}}},
+ {apiVersion:"v1",kind:"Service",metadata:{name:"foreign-ns",namespace:"sandbox-ui",uid:"foreign2",labels:{"clerum.io/recipe-namespace":"other-namespace","clerum.io/recipe-name":"fixture"}}}
+]}' > "$TEST_ROOT/children.json"
+wrc_capture_recipe_children sandbox-recipes fixture recipe-uid
+jq -se 'map(.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/null
+echo 'PASS: generated ownerless/cross-namespace children captured without foreign resources'
+
+# Validate rejected IDs before any fixture directory or cluster operation.
+wrc_require_networkpolicy_lease() { return 0; }
+for invalid in 'UPPER' 'contains-dash' '../escape' '1234567890123'; do
+ E2E_RUN_ID="$invalid" must_reject wrc_fixture_init
+done
+echo 'PASS: invalid run IDs fail without lossy normalization'
+
+printf 'Forwarding from 127.0.0.1:38123 -> 8080\n' > "$TEST_ROOT/port-forward.log"
+[[ "$(wrc_port_forward_reported_port 8080)" == 38123 ]]
+must_reject wrc_port_forward_reported_port 8090
+printf 'Forwarding from [::1]:38123 -> 8080\n' > "$TEST_ROOT/port-forward.log"
+must_reject wrc_port_forward_reported_port 8080
+printf 'Forwarding from 127.0.0.1:70000 -> 8080\n' > "$TEST_ROOT/port-forward.log"
+must_reject wrc_port_forward_reported_port 8080
+printf 'Forwarding from 127.0.0.1:38123 -> 8080\nForwarding from 127.0.0.1:39123 -> 8080\n' > "$TEST_ROOT/port-forward.log"
+must_reject wrc_port_forward_reported_port 8080
+echo 'PASS: dynamic binding comes only from one valid matching IPv4 child report'
+
+# These process doubles exercise fail-closed cleanup without launching a
+# process, touching user forwards, or relying on machine-specific ps output.
+WRC_PORT_FORWARD_PID=99999
+WRC_PORT_FORWARD_START=original
+WRC_PORT_FORWARD_COMMAND='kubectl --context=owned -n sandbox-recipes port-forward --address=127.0.0.1 svc/fixture :8080'
+PROCESS_STATE=live
+PROCESS_START=original
+PROCESS_COMMAND="$WRC_PORT_FORWARD_COMMAND"
+SIGNALLED=0
+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" "$WRC_PORT_FORWARD_PID" \
+ "$WRC_PORT_FORWARD_START" owned owned "$ROOT" sandbox-recipes fixture 38123 8080
+pf_owner_process_state() { printf '%s\n' "$PROCESS_STATE"; }
+pf_owner_process_start() { printf '%s\n' "$PROCESS_START"; }
+pf_owner_process_command() { printf '%s\n' "$PROCESS_COMMAND"; }
+pf_owner_signal_process() { SIGNALLED=$((SIGNALLED + 1)); PROCESS_STATE=dead; }
+pf_owner_reap_process() { :; }
+wrc_assert_port_forward
+PROCESS_START=reused
+must_reject wrc_stop_port_forward
+[[ "$SIGNALLED" == 0 ]]
+PROCESS_START=original
+PROCESS_COMMAND='foreign process'
+must_reject wrc_stop_port_forward
+[[ "$SIGNALLED" == 0 ]]
+PROCESS_COMMAND="$WRC_PORT_FORWARD_COMMAND"
+PROCESS_STATE=dead
+must_reject wrc_stop_port_forward
+[[ "$SIGNALLED" == 0 ]]
+PROCESS_STATE=live
+wrc_stop_port_forward
+[[ "$SIGNALLED" == 1 && -z "$WRC_PORT_FORWARD_PID" ]]
+[[ ! -e "$TEST_ROOT/port-forward.pid" ]] || {
+ echo 'FAIL: normal exit left the dead owned port-forward record' >&2
+ exit 1
+}
+echo 'PASS: dead/reused/foreign forwards fail; exact live child is stopped and reaped'
+
+WRC_PORT_FORWARD_PID=99999
+SIGNALLED=0
+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" "$WRC_PORT_FORWARD_PID" \
+ "$WRC_PORT_FORWARD_START" owned owned "$ROOT" sandbox-recipes fixture 38123 8080
+printf '0\n' > "$TEST_ROOT/process-reads"
+pf_owner_process_state() {
+ if [[ "$SIGNALLED" == 0 ]]; then
+ printf 'live\n'
+ return
+ fi
+ local reads
+ reads="$(cat "$TEST_ROOT/process-reads")"
+ printf '%s\n' "$((reads + 1))" > "$TEST_ROOT/process-reads"
+ if [[ "$reads" -lt 2 ]]; then printf 'live\n'; else printf 'dead\n'; fi
+}
+pf_owner_process_start() {
+ # Model death between the post-TERM state check and the identity read.
+ [[ "$SIGNALLED" == 0 ]] || return 1
+ printf '%s\n' "$PROCESS_START"
+}
+wrc_stop_port_forward
+[[ "$SIGNALLED" == 1 && -z "$WRC_PORT_FORWARD_PID" && ! -e "$TEST_ROOT/port-forward.pid" ]]
+echo 'PASS: death during post-TERM identity read also removes the owned record'
+
+WRC_PORT_FORWARD_PID=99999
+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" 99998 original \
+ owned owned "$ROOT" sandbox-recipes fixture 38123 8080
+must_reject wrc_reap_port_forward
+[[ -e "$TEST_ROOT/port-forward.pid" && "$WRC_PORT_FORWARD_PID" == 99999 ]]
+rm "$TEST_ROOT/port-forward.pid"
+pf_owner_write_record_atomic "$TEST_ROOT/port-forward.pid" 99999 foreign-start \
+ owned owned "$ROOT" sandbox-recipes fixture 38123 8080
+must_reject wrc_reap_port_forward
+[[ -e "$TEST_ROOT/port-forward.pid" && "$WRC_PORT_FORWARD_PID" == 99999 ]]
+echo 'PASS: cleanup preserves records with a different PID or process start'
+
+echo 'WRC_FIXTURES_CONTRACT_PASS'
diff --git a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
index 7a6c11298..087d02ec9 100755
--- a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
+++ b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
@@ -1,19 +1,27 @@
#!/usr/bin/env bash
-set -u
+# Hermetic orchestration contract. These receipts prove aggregate execution and
+# failure propagation, not NetworkPolicy convergence or real dataplane traffic.
+set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
AGGREGATE="${ROOT}/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh"
-ROUTES="${ROOT}/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh"
-INTDEP="${ROOT}/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh"
-OAUTH="${ROOT}/scripts/e2e/e2e-sandbox-ui-oauth.sh"
-WEBHOOK="${ROOT}/scripts/e2e/e2e-webhooks-basic.sh"
-HELPER="${ROOT}/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"
+FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-np-aggregate-contract.XXXXXX")"
+trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT
+
+suites=(
+ e2e-wrc-networkpolicy-service-routes.sh
+ e2e-wrc-internal-dependency-networkpolicy.sh
+ e2e-sandbox-ui-oauth.sh
+ e2e-webhooks-basic.sh
+)
+CONTEXT='owned-profile-contract'
FAIL=0
pass() { printf 'PASS: %s\n' "$1"; }
-fail() { printf 'FAIL: %s\n' "$1"; FAIL=1; }
+fail() { printf 'FAIL: %s\n' "$1" >&2; FAIL=1; }
-for script in "$AGGREGATE" "$ROUTES" "$INTDEP" "$OAUTH" "$WEBHOOK" "$HELPER"; do
+for script in "$AGGREGATE" "${suites[@]/#/${ROOT}/scripts/e2e/}" \
+ "${ROOT}/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"; do
if bash -n "$script"; then
pass "$(basename "$script") has valid bash syntax"
else
@@ -21,128 +29,132 @@ for script in "$AGGREGATE" "$ROUTES" "$INTDEP" "$OAUTH" "$WEBHOOK" "$HELPER"; do
fi
done
-for suite in \
- e2e-wrc-networkpolicy-service-routes.sh \
- e2e-wrc-internal-dependency-networkpolicy.sh \
- e2e-sandbox-ui-oauth.sh \
- e2e-webhooks-basic.sh; do
- grep -Fq "$suite" "$AGGREGATE" || fail "aggregate omits $suite"
-done
-if grep -Fq 'WRC_NETWORKPOLICY_E2E_PASS' "$AGGREGATE" &&
- grep -Fq 'set -euo pipefail' "$AGGREGATE"; then
- pass "aggregate is fail-loud and emits a terminal suite count"
-else
- fail "aggregate can pass without executing every suite"
-fi
-
+# This is a static wiring check only. Behaviour is exercised below.
if grep -Fq 'e2e-wrc-networkpolicy-live-convergence.sh' "${ROOT}/Makefile"; then
- pass "canonical Make E2E aggregate includes the WRC NetworkPolicy gate"
+ pass 'Makefile references the aggregate entry point (static wiring)'
else
- fail "canonical Make E2E aggregate omits the WRC NetworkPolicy gate"
+ fail 'Makefile does not reference the aggregate entry point'
fi
-if grep -Fq 'patch workflowrecipe' "$HELPER" &&
- grep -Fq 'metadata.generation' "$HELPER" &&
- grep -Fq 'deployment/${controller_deployment}' "$HELPER"; then
- pass "reconcile trigger advances the parent spec and observes the real WRC pass"
-else
- fail "reconcile trigger does not prove a parent recipe reconcile"
-fi
+mkdir -p "$FIXTURE_ROOT/scripts/e2e/_lib"
+cp "$AGGREGATE" "$FIXTURE_ROOT/aggregate-original.sh"
-if grep -Fq 'network policy unchanged; skipping update' "$HELPER" &&
- grep -Fq 'network policy egress set unchanged; skipping live apply' "$HELPER" &&
- grep -Fq '.policy == $policy' "$HELPER" &&
- grep -Fq '.namespace == $namespace' "$HELPER" &&
- grep -Fq '.family == $family' "$HELPER"; then
- pass "no-churn witness binds structured WRC output to one policy and family"
-else
- fail "no-churn witness can pass without proving the named policy reached its no-op path"
+# Only the fixture copy substitutes the external runtime/lease boundary. The
+# production aggregate has no flag or environment variable to bypass it.
+cat > "$FIXTURE_ROOT/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh" <<'STUB'
+wrc_require_networkpolicy_lease() {
+ : "${KUBECONTEXT:?KUBECONTEXT is required}"
+ printf 'lease|%s\n' "$KUBECONTEXT" >> "$CONTRACT_RECEIPTS"
+ return "${CONTRACT_LEASE_EXIT:-0}"
+}
+STUB
+
+write_children() {
+ local suite
+ for suite in "${suites[@]}"; do
+ cat > "$FIXTURE_ROOT/scripts/e2e/$suite" <<'CHILD'
+#!/usr/bin/env bash
+set -euo pipefail
+suite="${0##*/}"
+printf 'child|%s|%s\n' "$suite" "${KUBECONTEXT-<missing>}" >> "$CONTRACT_RECEIPTS"
+if [ "$suite" = "${CONTRACT_FAIL_SUITE:-}" ]; then
+ exit "$CONTRACT_CHILD_EXIT"
fi
+CHILD
+ done
+}
-for entry in \
- "$ROUTES:service-route" \
- "$INTDEP:internal-dependency" \
- "$OAUTH:OAuth broker" \
- "$WEBHOOK:webhook"; do
- script=${entry%%:*}
- label=${entry#*:}
- trigger_count=$(grep -Fc 'wrc_trigger_recipe_reconcile' "$script" || true)
- if [ "$trigger_count" -ge 2 ]; then
- pass "${label} E2E drives both repair and equivalent no-op reconciles"
- else
- fail "${label} E2E does not drive both required reconciles"
+# Verify the exact receipts, not a name or counter appearing in source/output.
+# Prefix length also proves no child after a failed/missing child was executed.
+check_run() {
+ local variant="$1" label="$2" expected_status="$3" child_count="$4" context_mode="$5"
+ local failed_suite="${6:-}" child_exit="${7:-0}" lease_exit="${8:-0}" status=0 index marker_count
+ : > "$FIXTURE_ROOT/receipts"
+ : > "$FIXTURE_ROOT/expected"
+ if [ "$context_mode" = 'present' ]; then
+ printf 'lease|%s\n' "$CONTEXT" >> "$FIXTURE_ROOT/expected"
+ for ((index = 0; index < child_count; index++)); do
+ printf 'child|%s|%s\n' "${suites[$index]}" "$CONTEXT" >> "$FIXTURE_ROOT/expected"
+ done
fi
-done
-for entry in \
- "$ROUTES:3:service-route" \
- "$INTDEP:2:internal-dependency" \
- "$OAUTH:1:OAuth broker" \
- "$WEBHOOK:3:webhook"; do
- script=${entry%%:*}
- remainder=${entry#*:}
- minimum=${remainder%%:*}
- label=${remainder#*:}
- witness_count=$(grep -Fc 'wrc_wait_for_np_noop_witness' "$script" || true)
- if [ "$witness_count" -ge "$minimum" ]; then
- pass "${label} E2E requires a no-op witness for every in-scope policy"
- else
- fail "${label} E2E has ${witness_count}/${minimum} required no-op witnesses"
- fi
-done
+ local environment=(
+ env -i PATH=/usr/bin:/bin
+ "CONTRACT_RECEIPTS=$FIXTURE_ROOT/receipts"
+ "CONTRACT_FAIL_SUITE=$failed_suite" "CONTRACT_CHILD_EXIT=$child_exit"
+ "CONTRACT_LEASE_EXIT=$lease_exit"
+ )
+ case "$context_mode" in
+ present) environment+=("KUBECONTEXT=$CONTEXT") ;;
+ empty) environment+=(KUBECONTEXT=) ;;
+ absent) ;;
+ *) printf 'invalid contract context mode\n' >&2; return 1 ;;
+ esac
+ "${environment[@]}" bash "$FIXTURE_ROOT/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh" \
+ > "$FIXTURE_ROOT/output" 2>&1 || status=$?
-for entry in "$OAUTH:OAuth broker" "$WEBHOOK:webhook"; do
- script=${entry%%:*}
- label=${entry#*:}
- if grep -Fq 'cleanup_status=1' "$script" &&
- grep -Fq 'cleanup also failed' "$script" &&
- grep -Fq 'status=1' "$script"; then
- pass "${label} E2E makes cleanup failure fail the aggregate"
- else
- fail "${label} E2E cleanup can be silently ignored"
+ marker_count="$(awk '/WRC_NETWORKPOLICY_E2E_PASS/ { count++ } END { print count+0 }' "$FIXTURE_ROOT/output")"
+ if [ "$status" -ne "$expected_status" ] ||
+ ! cmp -s "$FIXTURE_ROOT/expected" "$FIXTURE_ROOT/receipts" ||
+ { [ "$expected_status" -eq 0 ] &&
+ { [ "$marker_count" -ne 1 ] || [ "$(tail -n 1 "$FIXTURE_ROOT/output")" != 'WRC_NETWORKPOLICY_E2E_PASS suites=4' ]; }; } ||
+ { [ "$expected_status" -ne 0 ] && [ "$marker_count" -ne 0 ]; }; then
+ if [ "$variant" = original ]; then
+ printf 'FAIL: %s (exit=%s expected=%s, PASS markers=%s)\n' "$label" "$status" "$expected_status" "$marker_count" >&2
+ diff -u "$FIXTURE_ROOT/expected" "$FIXTURE_ROOT/receipts" >&2 || true
+ fi
+ return 1
fi
-done
+ if [ "$variant" = original ]; then pass "$label"; fi
+}
-for family in UI_INGRESS_POLICY WL_EGRESS_POLICY WL_INGRESS_POLICY; do
- grep -Fq "$family" "$ROUTES" || fail "service-route E2E omits $family"
-done
-if grep -Fq 'correctly configured positive routes' "$ROUTES" &&
- grep -Fq 'negative controls' "$ROUTES" &&
- grep -Fq 'live drift repair' "$ROUTES" &&
- grep -Fq 'terminating race self-heals' "$ROUTES" &&
- grep -Fq 'wrc_wait_for_np_recreated' "$ROUTES" &&
- grep -Fq 'unexpected second parent spec event' "$ROUTES" &&
- grep -Fq 'steady-state no-churn' "$ROUTES"; then
- pass "UI/workload route E2E proves positive, negative, repair, scheduled retry, and no-churn signals"
-else
- fail "UI/workload route E2E is missing a required signal"
-fi
+verify_aggregate() {
+ local variant="$1" rejected=0 index child_exit
+ cp "$FIXTURE_ROOT/aggregate-$variant.sh" "$FIXTURE_ROOT/scripts/e2e/e2e-wrc-networkpolicy-live-convergence.sh"
+ write_children
+ check_run "$variant" 'lease then four unique suites execute in order with the explicit context' 0 4 present || rejected=1
+ for ((index = 0; index < ${#suites[@]}; index++)); do
+ child_exit=$((41 + index))
+ check_run "$variant" "child $((index + 1)) failure preserves exit $child_exit and prevents later children/PASS" \
+ "$child_exit" "$((index + 1))" present "${suites[$index]}" "$child_exit" || rejected=1
+ done
+ for ((index = 0; index < ${#suites[@]}; index++)); do
+ rm -- "$FIXTURE_ROOT/scripts/e2e/${suites[$index]}"
+ check_run "$variant" "missing child $((index + 1)) fails before later children/PASS" 1 "$index" present || rejected=1
+ write_children
+ done
+ check_run "$variant" 'absent context fails before the lease or any child' 1 0 absent || rejected=1
+ check_run "$variant" 'empty context fails before the lease or any child' 1 0 empty || rejected=1
+ check_run "$variant" 'lease rejection propagates before any child or PASS' 67 0 present '' 0 67 || rejected=1
+ return "$rejected"
+}
-if grep -Fq 'Live drift repair and steady no-churn' "$INTDEP" &&
- grep -Fq 'Internal dependency traffic recovered' "$INTDEP"; then
- pass "internal-dependency E2E proves live repair and restored traffic"
-else
- fail "internal-dependency E2E lacks repair or traffic evidence"
-fi
+if ! verify_aggregate original; then fail 'real aggregate violates its execution contract'; fi
-if grep -Fq 'BACKGROUND_POLICY_NAME' "$OAUTH" &&
- grep -Fq 'Opted-in background workload reaches' "$OAUTH" &&
- grep -Fq 'Unlabelled sandbox workload cannot use' "$OAUTH" &&
- grep -Fq 'OAuth broker route recovered' "$OAUTH"; then
- pass "OAuth broker E2E proves allow, deny, repair, and no-churn behavior"
-else
- fail "OAuth broker E2E lacks a required dataplane signal"
-fi
-
-for policy in PROXY_INGRESS_POLICY HANDLER_EGRESS_POLICY HANDLER_INGRESS_POLICY; do
- grep -Fq "$policy" "$WEBHOOK" || fail "webhook E2E omits $policy"
+# Mutation controls prove the contract cannot be fooled by keeping the suite
+# names, executed counter, set -e, and terminal PASS text while removing the work
+# or masking child failure. They mutate only a disposable aggregate copy.
+for mutation in skipped-child ignored-error; do
+ if ! awk -v mutation="$mutation" '
+ /KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/ {
+ count++
+ if (mutation == "skipped-child") {
+ sub(/KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/, ": # skipped child")
+ } else {
+ sub(/KUBECONTEXT="\$KUBECONTEXT" bash "\$path"/, "KUBECONTEXT=\"$KUBECONTEXT\" bash \"$path\" || true")
+ }
+ }
+ { print }
+ END { if (count != 1) exit 1 }
+ ' "$FIXTURE_ROOT/aggregate-original.sh" > "$FIXTURE_ROOT/aggregate-$mutation.sh"; then
+ fail "cannot construct $mutation mutation; update its injection site"
+ continue
+ fi
+ if verify_aggregate "$mutation"; then
+ fail "contract accepted $mutation despite preserved names/counter/PASS"
+ else
+ pass "contract rejects $mutation despite preserved names/counter/PASS"
+ fi
done
-if grep -Fq 'live drift and owner repair' "$WEBHOOK" &&
- grep -Fq 'Signed webhook route recovered' "$WEBHOOK" &&
- grep -Fq 'stayed resourceVersion-stable' "$WEBHOOK"; then
- pass "webhook E2E proves all three policies, owner repair, route recovery, and no churn"
-else
- fail "webhook E2E lacks a required convergence signal"
-fi
exit "$FAIL"
diff --git a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh
new file mode 100755
index 000000000..14e0c3cc4
--- /dev/null
+++ b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh
@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+# Contract tests execute the real HTTP probe and observation decisions. Only
+# Kubernetes transport, the remote nc process, and elapsed time are fixtures.
+# Functions are dispatched through expect_pass/expect_reject and the sourced helper.
+# shellcheck disable=SC2329
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-np-probe-contract.XXXXXX")"
+trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT
+# shellcheck source=../e2e/_lib/wrc-networkpolicy-convergence.sh
+source "$ROOT/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh"
+
+FAIL=0
+POLL_INTERVAL=1
+export E2E_PROBE_TIMEOUT=2
+export PROBE_SCENARIO=http
+export PROBE_SEQUENCE_COUNTER="$FIXTURE_ROOT/nc-counter"
+PROBE_TRANSPORT=http
+OBSERVATION_SCENARIO=clean
+ok() { :; }
+fail() { printf '%s\n' "$*" >&2; }
+
+expect_pass() {
+ local label=$1
+ shift
+ if "$@" > "$FIXTURE_ROOT/result" 2>&1; then
+ printf 'PASS: %s\n' "$label"
+ else
+ printf 'FAIL: %s\n' "$label" >&2
+ cat "$FIXTURE_ROOT/result" >&2
+ FAIL=1
+ fi
+}
+
+expect_reject() {
+ local label=$1
+ shift
+ if "$@" > "$FIXTURE_ROOT/result" 2>&1; then
+ printf 'FAIL: accepted %s\n' "$label" >&2
+ cat "$FIXTURE_ROOT/result" >&2
+ FAIL=1
+ else
+ printf 'PASS: rejects %s\n' "$label"
+ fi
+}
+
+mkdir -p "$FIXTURE_ROOT/tools" "$FIXTURE_ROOT/missing-tools"
+for tool in grep base64 tr mktemp rm; do
+ ln -s "$(command -v "$tool")" "$FIXTURE_ROOT/tools/$tool"
+done
+cat > "$FIXTURE_ROOT/tools/nc" <<'NC'
+#!/bin/sh
+case "$PROBE_SCENARIO" in
+ timeout-then-http|http-then-timeout)
+ read -r count < "$PROBE_SEQUENCE_COUNTER"
+ count=$((count + 1))
+ printf '%s\n' "$count" > "$PROBE_SEQUENCE_COUNTER"
+ if { [ "$PROBE_SCENARIO" = timeout-then-http ] && [ "$count" -eq 1 ]; } ||
+ { [ "$PROBE_SCENARIO" = http-then-timeout ] && [ "$count" -gt 1 ]; }; then
+ printf 'nc: connect timed out\n' >&2
+ exit 1
+ fi
+ printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal'
+ exit 0 ;;
+esac
+case "$PROBE_SCENARIO" in
+ http) printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal' ;;
+ http-403) printf 'HTTP/1.0 403 Forbidden\r\n\r\nfixture-auth-denied' ;;
+ timeout) printf 'nc: connect timed out\n' >&2; exit 1 ;;
+ refused) printf 'nc: Connection refused\n' >&2; exit 1 ;;
+ empty) exit 0 ;;
+ malformed) printf 'not an HTTP response' ;;
+ *) exit 99 ;;
+esac
+NC
+chmod +x "$FIXTURE_ROOT/tools/nc"
+
+log_event() {
+ printf '%s %s\n' "$1" "$2" >> "$FIXTURE_ROOT/logs"
+}
+
+noop_event='{"msg":"network policy unchanged; skipping update","namespace":"sandbox-recipes","policy":"policy","family":"workload-ingress"}'
+complete_event='{"msg":"recipe reconciliation completed","recipe":"recipe","namespace":"sandbox-recipes","uid":"recipe-uid","generation":2,"phase":"active","requeueAfterMs":0}'
+write_event='{"msg":"network policy replaced","namespace":"sandbox-recipes","policy":"policy","family":"wrong-family"}'
+
+kctl() {
+ if [ "$1" = exec ]; then
+ printf 'exec\n' >> "$FIXTURE_ROOT/exec-calls"
+ case "$PROBE_TRANSPORT" in
+ exec-forbidden) printf 'forbidden: pods/exec\n' >&2; return 13 ;;
+ empty-exec) return 0 ;;
+ bad-nonce) printf 'WRC_NP_BEGIN|foreign\nWRC_NP_END|foreign|timeout|\n'; return 0 ;;
+ malformed-frame) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|unknown|\n' "${10}" "${10}"; return 0 ;;
+ malformed-base64) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|http|!!!!\n' "${10}" "${10}"; return 0 ;;
+ malformed-http) printf 'WRC_NP_BEGIN|%s\nWRC_NP_END|%s|http|bm90IEhUVFA=\n' "${10}" "${10}"; return 0 ;;
+ esac
+ while [ "$1" != -- ]; do shift; done
+ shift
+ [ "$1" = sh ] || return 99
+ shift
+ if [ "$PROBE_TRANSPORT" = missing-tool ]; then
+ PATH="$FIXTURE_ROOT/missing-tools" /bin/sh "$@"
+ else
+ PATH="$FIXTURE_ROOT/tools" /bin/sh "$@"
+ fi
+ return
+ fi
+ case "$1:$2" in
+ get:deployment) printf '%s\n' '{"spec":{"replicas":1,"selector":{"matchLabels":{"app":"wrc"}}}}' ;;
+ get:pods) cat "$FIXTURE_ROOT/pods" ;;
+ logs:*) cat "$FIXTURE_ROOT/logs" ;;
+ get:networkpolicy) cat "$FIXTURE_ROOT/policy" ;;
+ get:workflowrecipe)
+ if [[ "$*" == *'jsonpath='* ]]; then
+ cat "$FIXTURE_ROOT/generation"
+ else
+ printf '%s\n' '{"metadata":{"uid":"recipe-uid","generation":1}}'
+ fi ;;
+ patch:workflowrecipe) printf '2\n' > "$FIXTURE_ROOT/generation" ;;
+ *) printf 'unexpected fixture transport: %s\n' "$*" >&2; return 99 ;;
+ esac
+}
+
+# A fixed cursor and advancing elapsed clock keep this matrix bounded without
+# sleeping or weakening the production minimum observation window of one second.
+# Unsetting Bash's special SECONDS removes its wall-clock behaviour; assigning
+# zero alone would leave a timing race between the real clock and fixture ticks.
+unset SECONDS
+SECONDS=0
+wrc_now() { printf '2026-09-04T00:00:02.000000000Z\n'; }
+sleep() {
+ SECONDS=$((SECONDS + $1))
+ printf 'tick\n' >> "$FIXTURE_ROOT/ticks"
+ if [ "$OBSERVATION_SCENARIO" = delayed-write ]; then
+ log_event '2026-09-04T00:00:04.000000000Z' "$write_event"
+ elif [ "$OBSERVATION_SCENARIO" = observer-changed ]; then
+ printf '%s\n' '{"items":[{"metadata":{"name":"wrc","uid":"pod-uid"},"status":{"phase":"Running","containerStatuses":[{"name":"wrc","ready":true,"containerID":"container-new","restartCount":1}]}}]}' > "$FIXTURE_ROOT/pods"
+ fi
+}
+
+probe_allowed() {
+ wrc_assert_http_allowed 'fixture route' sandbox-recipes pod fixture 8080 fixture-business-signal
+}
+probe_blocked() {
+ wrc_assert_http_blocked 'fixture isolation' sandbox-recipes pod fixture 8080
+}
+
+expect_pass 'HTTP probe executes remote tools and validates the application response' probe_allowed
+expect_reject 'HTTP response as NetworkPolicy denial' probe_blocked
+PROBE_SCENARIO=http-403
+expect_reject 'HTTP authorization rejection as NetworkPolicy denial' probe_blocked
+PROBE_SCENARIO=timeout
+expect_pass 'completed remote connection timeout as a network observation' probe_blocked
+expect_reject 'timeout as an allowed route' probe_allowed
+for PROBE_SCENARIO in empty malformed refused; do
+ expect_reject "remote $PROBE_SCENARIO response as NetworkPolicy denial" probe_blocked
+done
+PROBE_SCENARIO=http
+for PROBE_TRANSPORT in exec-forbidden missing-tool empty-exec bad-nonce malformed-frame malformed-base64 malformed-http; do
+ expect_reject "$PROBE_TRANSPORT probe envelope" wrc_http_probe sandbox-recipes pod fixture 8080
+done
+PROBE_TRANSPORT=http
+
+probe_sequence() {
+ local scenario=$1 assertion=$2 count
+ PROBE_SCENARIO=$scenario
+ printf '0\n' > "$PROBE_SEQUENCE_COUNTER"
+ "$assertion" || return 1
+ read -r count < "$PROBE_SEQUENCE_COUNTER"
+ [ "$count" -eq 2 ]
+}
+
+executor_aborts_once() {
+ local assertion=$1 before=$SECONDS count
+ PROBE_TRANSPORT=exec-forbidden
+ : > "$FIXTURE_ROOT/exec-calls"
+ if "$assertion"; then return 1; fi
+ count="$(wc -l < "$FIXTURE_ROOT/exec-calls")"
+ [ "$count" -eq 1 ] && [ "$SECONDS" -eq "$before" ]
+}
+
+expect_pass 'allowed route waits through timeout then validates HTTP after CNI propagation' \
+ probe_sequence timeout-then-http probe_allowed
+expect_pass 'denied route waits through HTTP then observes timeout after CNI propagation' \
+ probe_sequence http-then-timeout probe_blocked
+expect_pass 'allowed assertion aborts once on exec failure without waiting/retrying' executor_aborts_once probe_allowed
+expect_pass 'denied assertion aborts once on exec failure without waiting/retrying' executor_aborts_once probe_blocked
+PROBE_SCENARIO=http
+PROBE_TRANSPORT=http
+
+observe() {
+ local scenario=$1
+ OBSERVATION_SCENARIO=$scenario
+ : > "$FIXTURE_ROOT/logs"
+ : > "$FIXTURE_ROOT/ticks"
+ printf '1\n' > "$FIXTURE_ROOT/generation"
+ printf '%s\n' '{"metadata":{"uid":"policy-uid","resourceVersion":"1"}}' > "$FIXTURE_ROOT/policy"
+ printf '%s\n' '{"items":[{"metadata":{"name":"wrc","uid":"pod-uid"},"status":{"phase":"Running","containerStatuses":[{"name":"wrc","ready":true,"containerID":"container-one","restartCount":0}]}}]}' > "$FIXTURE_ROOT/pods"
+ wrc_begin_np_observation || return 1
+ wrc_track_np sandbox-recipes policy workload-ingress || return 1
+ if [ "$scenario" = pretrigger-rv-drift ]; then
+ printf '%s\n' '{"metadata":{"uid":"policy-uid","resourceVersion":"2"}}' > "$FIXTURE_ROOT/policy"
+ fi
+ case "$scenario" in
+ write-before)
+ log_event '2026-09-04T00:00:02.100000000Z' "$write_event"
+ log_event '2026-09-04T00:00:03.000000000Z' "$noop_event" ;;
+ write-after)
+ log_event '2026-09-04T00:00:03.000000000Z' "$noop_event"
+ log_event '2026-09-04T00:00:03.100000000Z' "$write_event" ;;
+ wrong-family)
+ log_event '2026-09-04T00:00:03.000000000Z' "${noop_event/workload-ingress/oauth-broker-egress}" ;;
+ missing-witness) ;;
+ old-logs) log_event '2026-09-04T00:00:01.999999999Z' "$noop_event" ;;
+ *) log_event '2026-09-04T00:00:03.000000000Z' "$noop_event" ;;
+ esac
+ log_event '2026-09-04T00:00:03.200000000Z' "$complete_event"
+ wrc_trigger_recipe_reconcile sandbox-recipes recipe 3 || return 1
+ SECONDS=0
+ wrc_assert_np_observation_clean 1 3 || return 1
+ # An initial no-op is insufficient: success must span the full window.
+ [ -s "$FIXTURE_ROOT/ticks" ]
+}
+
+expect_pass 'complete no-write window with an exact active reconcile and stable observer/UID/RV' observe clean
+for scenario in write-before write-after delayed-write wrong-family missing-witness old-logs observer-changed pretrigger-rv-drift; do
+ expect_reject "$scenario despite a completed parent reconcile" observe "$scenario"
+done
+
+exit "$FAIL"
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index a099d73c4..fde522079 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.345",
+ "version": "0.1.346",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.345",
+ "version": "0.1.346",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index 474d7ac77..af9e20fcd 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.345",
+ "version": "0.1.346",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts
index e708077f9..c42e1cfb6 100644
--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts
+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.test.ts
@@ -133,6 +133,7 @@ describe('NetworkPolicy live convergence', () => {
it('makes workload-egress metadata drift bypass its temporal prefilter', () => {
const desired = policy()
const live = structuredClone(desired)
+ desired.metadata!.annotations = { 'clerum.io/egress-fqdn-resolved-at': 'new-timestamp' }
live.metadata!.annotations = { 'clerum.io/egress-fqdn-resolved-at': 'old-timestamp' }
expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(true)
@@ -157,6 +158,21 @@ describe('NetworkPolicy live convergence', () => {
expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)
})
+ it('compares stable workload-egress annotations and removal of owned state', () => {
+ const desired = policy()
+ desired.metadata!.annotations = { 'clerum.io/test-contract': 'current' }
+ const live = structuredClone(desired)
+ live.metadata!.annotations!['clerum.io/test-contract'] = 'stale'
+ expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)
+
+ live.metadata!.annotations = {
+ ...desired.metadata!.annotations,
+ 'clerum.io/egress-fqdn-state': 'stale',
+ }
+ expect(networkPolicyMetadataMatchesDesired(desired, live)).toBe(false)
+ expect(networkPolicyMatchesDesired(desired, live)).toBe(false)
+ })
+
it('repairs a stale or missing gateway owner only for the same WRC recipe identity', () => {
const desired = gatewayPolicy('uid-new')
const stale = gatewayPolicy('uid-old')
diff --git a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
index ee5ad05d7..d37c830fd 100644
--- a/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
+++ b/workflow-recipes/src/reconciler/networkPolicyConvergence.ts
@@ -1,4 +1,9 @@
import * as k8s from '@kubernetes/client-node'
+import {
+ RESOLVED_AT_ANNOTATION,
+ STATE_ANNOTATION,
+ TARGETS_ANNOTATION,
+} from '@clerum/network-policy-core'
import { SPEC_HASH_ANNOTATION } from './specHash'
export type NetworkPolicyFamily =
@@ -41,6 +46,64 @@ const GATEWAY_IDENTITY_LABELS = [
'clerum.io/webhook-gateway',
] as const
+// These keys remain WRC-owned when external egress is removed from desired.
+// Other annotation namespaces are not ownership boundaries: preserve every
+// unauthored key, including keys authored by other clerum.io controllers.
+const EGRESS_STATE_ANNOTATIONS = new Set([
+ RESOLVED_AT_ANNOTATION,
+ STATE_ANNOTATION,
+ TARGETS_ANNOTATION,
+])
+
+function projectOwnedAnnotations(
+ desired: Record<string, string> | undefined,
+ existing: Record<string, string> | undefined,
+ temporal = false
+): Record<string, string> {
+ const keys = new Set([...Object.keys(desired ?? {}), ...EGRESS_STATE_ANNOTATIONS])
+ return Object.fromEntries(
+ [...keys].flatMap(key => {
+ if (key === SPEC_HASH_ANNOTATION) return []
+ // The accumulator owns renewal cadence for present temporal state. Its
+ // removal still has to reach apply, even when the enforced rules match.
+ if (temporal && EGRESS_STATE_ANNOTATIONS.has(key) && desired?.[key] !== undefined) return []
+ return existing?.[key] === undefined ? [] : [[key, existing[key]]]
+ })
+ )
+}
+
+/** Build a PUT from the same live snapshot used for ownership and drift checks. */
+export function buildNetworkPolicyReplacement(
+ desired: k8s.V1NetworkPolicy,
+ existing: k8s.V1NetworkPolicy
+): k8s.V1NetworkPolicy {
+ const replacement = structuredClone(desired)
+ const liveMetadata = existing.metadata ?? {}
+ const annotations = { ...liveMetadata.annotations }
+ for (const key of EGRESS_STATE_ANNOTATIONS) delete annotations[key]
+ Object.assign(annotations, desired.metadata?.annotations)
+ delete annotations[SPEC_HASH_ANNOTATION]
+ replacement.metadata = {
+ ...replacement.metadata,
+ labels: { ...liveMetadata.labels, ...replacement.metadata?.labels },
+ annotations,
+ ...(liveMetadata.finalizers?.length || replacement.metadata?.finalizers?.length
+ ? {
+ finalizers: [
+ ...new Set([
+ ...(liveMetadata.finalizers ?? []),
+ ...(replacement.metadata?.finalizers ?? []),
+ ]),
+ ],
+ }
+ : {}),
+ resourceVersion: liveMetadata.resourceVersion,
+ }
+ // ownerReferences intentionally come only from desired after the ownership
+ // veto. Foreign lifecycle owners must not be adopted, retained, or erased.
+ return replacement
+}
+
function canonicalizeValue(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonicalizeValue)
if (value && typeof value === 'object') {
@@ -54,13 +117,12 @@ function canonicalizeValue(value: unknown): unknown {
function projectAuthoredMap(
desired: Record<string, string> | undefined,
- existing: Record<string, string> | undefined,
- excludedKeys: ReadonlySet<string> = new Set()
+ existing: Record<string, string> | undefined
): Record<string, string> {
return Object.fromEntries(
- Object.keys(desired ?? {})
- .filter(key => !excludedKeys.has(key))
- .flatMap(key => (existing?.[key] === undefined ? [] : [[key, existing[key]]]))
+ Object.keys(desired ?? {}).flatMap(key =>
+ existing?.[key] === undefined ? [] : [[key, existing[key]]]
+ )
)
}
@@ -77,10 +139,9 @@ function projectNetworkPolicyForComparison(
name: candidateMetadata.name,
namespace: candidateMetadata.namespace,
labels: projectAuthoredMap(desiredMetadata.labels, candidateMetadata.labels),
- annotations: projectAuthoredMap(
+ annotations: projectOwnedAnnotations(
desiredMetadata.annotations,
- candidateMetadata.annotations,
- new Set([SPEC_HASH_ANNOTATION])
+ candidateMetadata.annotations
),
...(desiredMetadata.ownerReferences?.length
? { ownerReferences: candidateMetadata.ownerReferences }
@@ -145,8 +206,8 @@ export function networkPolicyMatchesDesired(
/**
* Compare the stable identity metadata that the workload-egress prefilter must
- * never hide. Temporal DNS/state annotations are intentionally excluded: their
- * persistence cadence is decided separately by the accumulator, while labels,
+ * never hide. Present temporal DNS/state values follow the accumulator's
+ * persistence cadence; retired state keys, stable annotations, labels,
* lifecycle owner and a terminating object must still reach the live apply.
*/
export function networkPolicyMetadataMatchesDesired(
@@ -164,6 +225,11 @@ export function networkPolicyMetadataMatchesDesired(
name: policy.metadata?.name,
namespace: policy.metadata?.namespace,
labels: projectAuthoredMap(desired.metadata?.labels, policy.metadata?.labels),
+ annotations: projectOwnedAnnotations(
+ desired.metadata?.annotations,
+ policy.metadata?.annotations,
+ true
+ ),
ownerReferences: desired.metadata?.ownerReferences?.length
? policy.metadata?.ownerReferences
: undefined,
@@ -219,6 +285,24 @@ export function classifyNetworkPolicyOwnership(
const desiredOwner = desiredOwners[0]
if (!desiredOwner.uid) return { kind: 'conflict', reason: 'desired-owner-uid-missing' }
+ // Admission metadata may coexist, but an additional lifecycle owner changes
+ // garbage-collection authority. Only explicitly desired non-controller refs
+ // are permitted; never silently drop an unexpected owner during replacement.
+ const desiredOtherOwners = (desired.metadata?.ownerReferences ?? []).filter(
+ owner => owner.controller !== true
+ )
+ const existingOtherOwners = (existing.metadata?.ownerReferences ?? []).filter(
+ owner => owner.controller !== true
+ )
+ if (
+ existingOtherOwners.length !== desiredOtherOwners.length ||
+ existingOtherOwners.some(
+ owner => !desiredOtherOwners.some(wanted => exactOwnerMatch(wanted, owner))
+ )
+ ) {
+ return { kind: 'conflict', reason: 'owner-reference-mismatch' }
+ }
+
if (existingOwners.length === 1 && exactOwnerMatch(desiredOwner, existingOwners[0])) {
return { kind: 'owned' }
}
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 7980d176b..11747dabe 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -859,6 +859,134 @@ describe('WorkflowRecipeReconciler', () => {
},
})
+ const families: NetworkPolicyFamily[] = [
+ 'ui-ingress',
+ 'workload-ingress',
+ 'workload-egress',
+ 'internal-dependency',
+ 'oauth-broker-egress',
+ 'webhook-gateway',
+ ]
+ const policyForFamily = (family: NetworkPolicyFamily) =>
+ family === 'webhook-gateway'
+ ? gatewayPolicy()
+ : family === 'internal-dependency'
+ ? intdepPolicy()
+ : plainPolicy()
+
+ it.each(families)(
+ 'preserves external metadata while repairing real %s spec drift',
+ async family => {
+ const desired = policyForFamily(family)
+ desired.metadata!.labels!['clerum.io/test-contract'] = 'current'
+ desired.metadata!.annotations = {
+ 'clerum.io/test-contract': 'current',
+ [SPEC_HASH]: 'obsolete',
+ }
+ const live = livePolicy(desired)
+ live.metadata!.labels!['clerum.io/test-contract'] = 'stale'
+ live.metadata!.labels!['admission.example/tier'] = 'audited'
+ live.metadata!.annotations = {
+ 'admission.example/audit': 'keep',
+ 'clerum.io/test-contract': 'stale',
+ [SPEC_HASH]: 'legacy',
+ 'clerum.io/egress-fqdn-state': 'stale-state',
+ 'clerum.io/egress-fqdn-targets': 'stale-targets',
+ 'clerum.io/egress-fqdn-resolved-at': 'stale-time',
+ }
+ live.metadata!.finalizers = ['admission.example/cleanup']
+ live.spec!.podSelector = {}
+ const desiredBefore = clone(desired)
+ const liveBefore = clone(live)
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await apply(desired, family)
+
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const body = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(body.metadata.labels).toEqual({
+ ...desired.metadata!.labels,
+ 'admission.example/tier': 'audited',
+ })
+ expect(body.metadata.annotations).toEqual({
+ 'admission.example/audit': 'keep',
+ 'clerum.io/test-contract': 'current',
+ })
+ expect(body.metadata.finalizers).toEqual(['admission.example/cleanup'])
+ expect(body.metadata.ownerReferences).toEqual(desired.metadata!.ownerReferences)
+ expect(body.metadata.resourceVersion).toBe('9')
+ expect(body.spec).toEqual(desired.spec)
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(desired).toEqual(desiredBefore)
+ expect(live).toEqual(liveBefore)
+ }
+ )
+
+ it.each(families)(
+ 'performs zero create/replace calls for equivalent %s with external metadata',
+ async family => {
+ const desired = policyForFamily(family)
+ const live = livePolicy(desired)
+ live.metadata!.labels!['admission.example/tier'] = 'audited'
+ live.metadata!.annotations = { 'admission.example/audit': 'keep', [SPEC_HASH]: 'legacy' }
+ live.metadata!.finalizers = ['admission.example/cleanup']
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await apply(desired, family)
+
+ expect(mockNetworkingApi.readNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ }
+ )
+
+ it('vetoes an additional non-controller owner instead of removing it on gateway replace', async () => {
+ const desired = gatewayPolicy()
+ const live = livePolicy(desired)
+ live.metadata!.ownerReferences!.push({
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ name: 'foreign',
+ uid: 'foreign',
+ })
+ live.spec!.podSelector = {}
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockResolvedValue(live)
+
+ await expect(apply(desired, 'webhook-gateway')).rejects.toMatchObject({
+ name: 'NetworkPolicyOwnershipConflictError',
+ })
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+
+ it('removes retired workload-egress state once without churning external metadata', async () => {
+ const desired = plainPolicy()
+ desired.metadata!.name = 'wl-egress-r-w'
+ const live = livePolicy(desired, {
+ annotations: {
+ 'admission.example/audit': 'keep',
+ 'clerum.io/egress-fqdn-state': 'retired',
+ 'clerum.io/egress-fqdn-targets': 'retired',
+ 'clerum.io/egress-fqdn-resolved-at': 'retired',
+ },
+ })
+ const priv = reconciler as unknown as {
+ egressWriteNeeded: (existing: k8s.V1NetworkPolicy, desired: k8s.V1NetworkPolicy) => boolean
+ }
+ expect(priv.egressWriteNeeded(live, desired)).toBe(true)
+ await apply(desired, 'workload-egress', live)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(1)
+ const updated = mockNetworkingApi.replaceNamespacedNetworkPolicy.mock.calls[0][0].body
+ expect(updated.metadata.annotations).toEqual({ 'admission.example/audit': 'keep' })
+ mockNetworkingApi.replaceNamespacedNetworkPolicy.mockClear()
+
+ expect(priv.egressWriteNeeded(updated, desired)).toBe(false)
+ await apply(desired, 'workload-egress', updated)
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+
function captureNetworkPolicyLogs(): {
entries: Array<Record<string, unknown>>
restore: () => void
@@ -1080,7 +1208,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(warning).toEqual(
expect.objectContaining({
level: 'warn',
- errorName: 'UnknownError',
+ errorName: 'NetworkPolicyInfraError',
errorCode: 500,
retryable: true,
})
@@ -3056,6 +3184,136 @@ describe('WorkflowRecipeReconciler', () => {
)
})
+ describe('workload egress pre-read infrastructure failures', () => {
+ it.each([
+ { workflow: true, code: 503 },
+ { workflow: true, code: 'ECONNRESET' },
+ { workflow: false, code: 503 },
+ { workflow: false, code: 'ECONNRESET' },
+ { workflow: true, code: 403 },
+ { workflow: false, code: 403 },
+ ])('classifies $code after successful DNS (workflow=$workflow)', async ({ workflow, code }) => {
+ const recipe = makeRecipe({
+ spec: {
+ workloads: [
+ {
+ id: 'worker',
+ type: 'deployment',
+ image: 'worker:latest',
+ port: 8080,
+ egressBindings: [{ dns: 'api.example.com', port: 443 }],
+ },
+ ],
+ ...(workflow ? { steps: [{ id: 'run', run: snippetRun() }] } : {}),
+ },
+ ...(!workflow
+ ? {
+ status: {
+ phase: 'active' as const,
+ message: 'Healthy before read failure',
+ workloads: [],
+ },
+ }
+ : {}),
+ })
+ const lookup = vi.fn(async () => ({
+ kind: 'ok' as const,
+ ipv4: ['93.184.216.10'],
+ ipv6: [],
+ ttlSeconds: 300,
+ }))
+ const rec = new WorkflowRecipeReconciler(new k8s.KubeConfig(), undefined, {
+ fqdnLookup: lookup,
+ })
+ const workflowReconcile = vi.fn()
+ ;(rec as unknown as { workflowReconciler: unknown }).workflowReconciler = {
+ reconcile: workflowReconcile,
+ validateWorkflowSpec: () => undefined,
+ }
+ const failure = { code }
+ mockNetworkingApi.readNamespacedNetworkPolicy.mockImplementation(async ({ name }) => {
+ if (name === 'wl-egress-test-recipe-worker') {
+ expect(lookup).toHaveBeenCalled()
+ throw failure
+ }
+ throw { code: 404 }
+ })
+
+ const result = await rec.reconcile(recipe)
+
+ expect(lookup).toHaveBeenCalledTimes(1)
+ if (code === 403) {
+ expect(result.phase).toBe('failed')
+ expect(result.requeueAfterMs).toBeUndefined()
+ } else {
+ expect(result.phase).toBe(workflow ? 'degraded' : 'active')
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
+ if (!workflow) expect(result.skipStatusPatch).toBe(true)
+ }
+ expect(workflowReconcile).not.toHaveBeenCalled()
+ expect(mockNetworkingApi.createNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).toHaveBeenCalledTimes(0)
+ })
+ })
+
+ describe('reconciliation completion receipt', () => {
+ it.each(['success', 'reconcile-throw', 'projection-throw'] as const)(
+ 'emits completion only after the full pipeline returns (%s)',
+ async outcome => {
+ const recipe = makeRecipe()
+ recipe.metadata.generation = 7
+ const previousLevel = process.env.LOG_LEVEL
+ process.env.LOG_LEVEL = 'info'
+ const receipts: Array<Record<string, unknown>> = []
+ const write = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown) => {
+ for (const line of String(chunk).split('\n')) {
+ if (!line.startsWith('{')) continue
+ const record = JSON.parse(line)
+ if (record.msg === 'recipe reconciliation completed') receipts.push(record)
+ }
+ return true
+ }) as typeof process.stdout.write)
+ const priv = reconciler as unknown as {
+ reconcileInternal: () => Promise<unknown>
+ projectPluginWorkloadSdk: () => unknown
+ }
+ const failure = new Error('pipeline failure')
+ const injected =
+ outcome === 'reconcile-throw'
+ ? vi.spyOn(priv, 'reconcileInternal').mockRejectedValue(failure)
+ : outcome === 'projection-throw'
+ ? vi.spyOn(priv, 'projectPluginWorkloadSdk').mockImplementation(() => {
+ throw failure
+ })
+ : undefined
+ try {
+ if (outcome === 'success') {
+ const result = await reconciler.reconcile(recipe)
+ expect(result.phase).toBe('active')
+ expect(receipts).toEqual([
+ expect.objectContaining({
+ recipe: 'test-recipe',
+ namespace: 'sandbox-recipes',
+ uid: 'uid-123',
+ generation: 7,
+ phase: 'active',
+ requeueAfterMs: 0,
+ }),
+ ])
+ } else {
+ await expect(reconciler.reconcile(recipe)).rejects.toBe(failure)
+ expect(receipts).toEqual([])
+ }
+ } finally {
+ injected?.mockRestore()
+ write.mockRestore()
+ if (previousLevel === undefined) delete process.env.LOG_LEVEL
+ else process.env.LOG_LEVEL = previousLevel
+ }
+ }
+ )
+ })
+
it('fails non-transport external egress in required mode when cluster enforcement is not confirmed', async () => {
const recipe = makeRecipe({
spec: {
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index bb72bea0e..e364bb611 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -91,6 +91,7 @@ import {
} from './mcpDelegation'
import {
type NetworkPolicyFamily,
+ buildNetworkPolicyReplacement,
decideNetworkPolicyConvergence,
networkPolicyMetadataMatchesDesired,
} from './networkPolicyConvergence'
@@ -1425,6 +1426,14 @@ export class WorkflowRecipeReconciler {
// both consumers treat as "no SDK opinion this pass".
const result = await this.reconcileInternal(recipe)
result.pluginWorkloadSdkProjection = this.projectPluginWorkloadSdk(recipe, result)
+ createLogger('wrc', recipe.metadata.name).info('recipe reconciliation completed', {
+ recipe: recipe.metadata.name,
+ namespace: recipe.metadata.namespace,
+ uid: recipe.metadata.uid,
+ generation: recipe.metadata.generation,
+ phase: result.phase,
+ requeueAfterMs: result.requeueAfterMs ?? 0,
+ })
return result
}
@@ -4759,6 +4768,14 @@ export class WorkflowRecipeReconciler {
return await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace })
} catch (error: unknown) {
if (getErrorCode(error) === 404) return null
+ if (isRetryableInfraError(error)) {
+ const safe = this.safeInfrastructureErrorFields(error)
+ throw new NetworkPolicyInfraError(
+ `NetworkPolicy "${name}" in ${namespace} read temporarily unavailable (${safe.errorCode ?? safe.errorName})`,
+ error,
+ safe.errorCode
+ )
+ }
throw error
}
}
@@ -5011,12 +5028,12 @@ export class WorkflowRecipeReconciler {
`NetworkPolicy "${policyName}" in ${namespace} has no resourceVersion`
)
}
- desired.metadata.resourceVersion = resourceVersion
+ const replacement = buildNetworkPolicyReplacement(desired, existing)
try {
await this.networkingApi.replaceNamespacedNetworkPolicy({
name: policyName,
namespace,
- body: desired,
+ body: replacement,
})
log.info('network policy replaced', {
...fields,
@@ -5073,22 +5090,11 @@ export class WorkflowRecipeReconciler {
for (let attempt = 1; attempt <= 2; attempt += 1) {
try {
return {
- existing: await this.networkingApi.readNamespacedNetworkPolicy({ name, namespace }),
+ existing: await this.readNetworkPolicyOrNull(name, namespace),
recoveredFromReadFailure: attempt > 1,
}
} catch (error: unknown) {
- if (getErrorCode(error) === 404) {
- return { existing: null, recoveredFromReadFailure: attempt > 1 }
- }
if (attempt === 2) {
- if (isRetryableInfraError(error)) {
- const safe = this.safeInfrastructureErrorFields(error)
- throw new NetworkPolicyInfraError(
- `NetworkPolicy "${name}" in ${namespace} read temporarily unavailable (${safe.errorCode ?? safe.errorName})`,
- error,
- safe.errorCode
- )
- }
throw error
}
log.warn('network policy read unavailable; retrying once', {