Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
#!/usr/bin/env bash
# Shared, fail-loud helpers for WRC NetworkPolicy live-convergence E2E gates.
# Source after e2e-lib.sh so kctl/ok/fail are available.
set -euo pipefail
WRC_NP_RECONCILE_BINDING='{}'
WRC_NP_OBSERVATION_SINCE=""
WRC_NP_OBSERVATION_PODS='[]'
WRC_NP_TRACKED='[]'
wrc_require_networkpolicy_lease() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd -P)"
if [ -z "${KUBECONTEXT:-}" ] || [ "$KUBECONTEXT" != "${MINIKUBE_PROFILE:-}" ] ||
[ "$KUBECONTEXT" != "${T2_CONTEXT:-${CONTROL_API_REAL_PG_CONTEXT:-}}" ]; then
printf 'PROFILE_LOCK_REQUIRED: WRC NetworkPolicy E2E needs the matching explicit owned profile/context\n' >&2
return 1
fi
bash "$root/scripts/minikube/require-t2-mutation-lock.sh"
}
wrc_now() {
python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'
}
# Track every controller replica and container incarnation, so rolling/restarted
# observers cannot silently lose the beginning of a no-write window.
wrc_controller_snapshot() {
local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
local deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes} definition selector replicas
definition="$(kctl get deployment "$deployment" -n "$namespace" -o json)" || return 1
selector="$(printf '%s' "$definition" | jq -er '
.spec.selector | select((.matchExpressions // [] | length) == 0)
| .matchLabels | select(length > 0) | to_entries | sort_by(.key)
| map(.key + "=" + .value) | join(",")')" || return 1
replicas="$(printf '%s' "$definition" | jq -er '.spec.replicas // 1 | select(. > 0)')" || return 1
kctl get pods -n "$namespace" -l "$selector" -o json | jq -ce --argjson replicas "$replicas" '
.items | select(length == $replicas)
| select(all(.[]; .metadata.deletionTimestamp == null and .status.phase == "Running"
and (.status.containerStatuses | length > 0)
and all(.status.containerStatuses[]; .ready == true and (.containerID | length > 0))))
| map({name:.metadata.name,uid:.metadata.uid,
containers:(.status.containerStatuses | sort_by(.name) | map({name,containerID,restartCount}))})
| sort_by(.name)'
}
wrc_controller_logs() {
local snapshot=$1 since=$2 pod logs
local namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
while IFS= read -r pod; do
[ -n "$pod" ] || return 1
logs="$(kctl logs "$pod" -n "$namespace" --all-containers=true --timestamps=true \
--since-time="$since")" || return 1
printf '%s\n' "$logs"
done < <(printf '%s' "$snapshot" | jq -r '.[].name')
}
# Kubernetes timestamps delimit the cursor; mixed legacy console lines are not
# JSON events. Rejecting writes is done over all events, never the first match.
wrc_log_events() {
local since=$1
jq -Rsc --arg since "$since" '
def timestamp_key:
sub("Z$"; "") | split(".")
| .[0] + "." + (((.[1] // "") + "000000000")[0:9]);
($since | timestamp_key) as $start
| split("\n") | map(
try (capture("^(?<at>[0-9T:.+-]+Z) (?<payload>.*)$")
| select((.at | timestamp_key) >= $start)
| .payload | fromjson | select(type == "object")) catch empty)'
}
wrc_np_spec_hash() {
local namespace=$1 name=$2
kctl get networkpolicy "$name" -n "$namespace" -o json |
python3 -c 'import hashlib,json,sys; spec=json.load(sys.stdin)["spec"]; payload=json.dumps(spec,sort_keys=True,separators=(",",":")).encode(); print(hashlib.sha256(payload).hexdigest())'
}
wrc_np_resource_version() {
local namespace=$1 name=$2
kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.resourceVersion}'
}
wrc_np_uid() {
local namespace=$1 name=$2
kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.uid}'
}
wrc_wait_for_np() {
local namespace=$1 name=$2 timeout=${3:-120} elapsed=0
while [ "$elapsed" -lt "$timeout" ]; do
if kctl get networkpolicy "$name" -n "$namespace" >/dev/null 2>&1; then
return 0
fi
sleep "$POLL_INTERVAL"
elapsed=$((elapsed + POLL_INTERVAL))
done
fail "Timed out waiting for NetworkPolicy ${namespace}/${name}"
return 1
}
wrc_wait_for_np_spec_hash() {
local namespace=$1 name=$2 expected=$3 timeout=${4:-120} elapsed=0 actual
while [ "$elapsed" -lt "$timeout" ]; do
actual="$(wrc_np_spec_hash "$namespace" "$name" 2>/dev/null || true)"
if [ -n "$actual" ] && [ "$actual" = "$expected" ]; then
return 0
fi
sleep "$POLL_INTERVAL"
elapsed=$((elapsed + POLL_INTERVAL))
done
fail "NetworkPolicy ${namespace}/${name} did not restore its baseline spec"
return 1
}
wrc_wait_for_np_recreated() {
local namespace=$1 name=$2 previous_uid=$3 expected_hash=$4 timeout=${5:-120} elapsed=0
local current_uid deletion_timestamp current_hash
while [ "$elapsed" -lt "$timeout" ]; do
current_uid="$(wrc_np_uid "$namespace" "$name" 2>/dev/null || true)"
deletion_timestamp="$(
kctl get networkpolicy "$name" -n "$namespace" \
-o jsonpath='{.metadata.deletionTimestamp}' 2>/dev/null || true
)"
current_hash="$(wrc_np_spec_hash "$namespace" "$name" 2>/dev/null || true)"
if [ -n "$current_uid" ] && [ "$current_uid" != "$previous_uid" ] &&
[ -z "$deletion_timestamp" ] && [ "$current_hash" = "$expected_hash" ]; then
ok "NetworkPolicy ${namespace}/${name} was recreated by the scheduled retry"
return 0
fi
sleep "$POLL_INTERVAL"
elapsed=$((elapsed + POLL_INTERVAL))
done
fail "NetworkPolicy ${namespace}/${name} was not recreated after its terminating race"
return 1
}
wrc_inject_selector_drift() {
local namespace=$1 name=$2
kctl patch networkpolicy "$name" -n "$namespace" --type=merge \
-p '{"spec":{"podSelector":{"matchLabels":{"e2e.clerum.io/forced-drift":"true"}}}}' \
>/dev/null
}
wrc_trigger_recipe_reconcile() {
local namespace=$1 name=$2 timeout=${3:-120}
local before_generation after_generation marker patch since_time started=$SECONDS
local recipe uid snapshot events logs
recipe="$(kctl get workflowrecipe "$name" -n "$namespace" -o json)" || return 1
before_generation="$(printf '%s' "$recipe" | jq -er '.metadata.generation')" || return 1
uid="$(printf '%s' "$recipe" | jq -er '.metadata.uid | select(length > 0)')" || return 1
[[ "$before_generation" =~ ^[0-9]+$ ]] || {
fail "WorkflowRecipe ${namespace}/${name} has no numeric generation before reconcile trigger"
return 1
}
WRC_NP_RECONCILE_SEQUENCE=${WRC_NP_RECONCILE_SEQUENCE:-0}
WRC_NP_RECONCILE_SEQUENCE=$((WRC_NP_RECONCILE_SEQUENCE + 1))
marker="np-e2e-${WRC_NP_RECONCILE_SEQUENCE}-$(date +%s)"
patch="$(jq -cn --arg description "$marker" '{spec:{description:$description}}')"
snapshot="$(wrc_controller_snapshot)" || return 1
since_time="$(wrc_now)"
kctl patch workflowrecipe "$name" -n "$namespace" --type=merge -p "$patch" >/dev/null
after_generation="$(
kctl get workflowrecipe "$name" -n "$namespace" -o jsonpath='{.metadata.generation}'
)"
if [[ ! "$after_generation" =~ ^[0-9]+$ ]] || [ "$after_generation" -le "$before_generation" ]; then
fail "WorkflowRecipe ${namespace}/${name} generation did not advance (${before_generation} -> ${after_generation})"
return 1
fi
WRC_NP_RECONCILE_BINDING="$(jq -cn --arg recipe "$name" --arg namespace "$namespace" \
--arg uid "$uid" --argjson generation "$after_generation" '{recipe:$recipe,namespace:$namespace,uid:$uid,generation:$generation}')"
while [ "$((SECONDS - started))" -lt "$timeout" ]; do
[ "$(wrc_controller_snapshot)" = "$snapshot" ] || {
fail "WRC observer changed during the parent reconcile"
return 1
}
logs="$(wrc_controller_logs "$snapshot" "$since_time")" || return 1
events="$(printf '%s' "$logs" | wrc_log_events "$since_time")" || return 1
if printf '%s' "$events" | jq -e --argjson binding "$WRC_NP_RECONCILE_BINDING" '
any(.[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe
and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation)' >/dev/null; then
ok "WorkflowRecipe ${namespace}/${name} completed reconcile at generation ${after_generation}"
return 0
fi
sleep "$POLL_INTERVAL"
done
fail "Timed out waiting for WRC to reconcile WorkflowRecipe ${namespace}/${name} generation ${after_generation}"
return 1
}
wrc_begin_np_observation() {
WRC_NP_OBSERVATION_SINCE="$(wrc_now)"
WRC_NP_OBSERVATION_PODS="$(wrc_controller_snapshot)" || return 1
WRC_NP_TRACKED='[]'
WRC_NP_RECONCILE_BINDING='{}'
}
wrc_track_np() {
local namespace=$1 name=$2 family=$3 mode=${4:-apply} policy
local message='network policy unchanged; skipping update'
[ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_RECONCILE_BINDING" = '{}' ] || {
fail 'NetworkPolicy baseline must be captured before the reconcile trigger'
return 1
}
if [ "$mode" = "workload-egress-prefilter" ]; then
message='network policy egress set unchanged; skipping live apply'
elif [ "$mode" != "apply" ]; then
fail "Unknown NetworkPolicy observation mode: ${mode}"
return 1
fi
policy="$(kctl get networkpolicy "$name" -n "$namespace" -o json)" || return 1
policy="$(printf '%s' "$policy" | jq -ce --arg namespace "$namespace" --arg policy "$name" \
--arg family "$family" --arg message "$message" '
select(.metadata.deletionTimestamp == null and (.metadata.uid | length > 0)
and (.metadata.resourceVersion | length > 0))
| {namespace:$namespace,policy:$policy,family:$family,message:$message,
uid:.metadata.uid,rv:.metadata.resourceVersion}')" || return 1
WRC_NP_TRACKED="$(printf '%s' "$WRC_NP_TRACKED" | jq -c --argjson policy "$policy" '. + [$policy]')"
}
wrc_assert_np_observation_clean() {
local seconds=${1:-20} timeout=${2:-120} started=$SECONDS logs events row current missing
[[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [[ "$timeout" =~ ^[1-9][0-9]*$ ]] &&
[ "$seconds" -le 300 ] && [ "$timeout" -le 600 ] && [ "$timeout" -ge "$seconds" ] || return 1
[ -n "$WRC_NP_OBSERVATION_SINCE" ] && [ "$WRC_NP_TRACKED" != '[]' ] &&
[ "$WRC_NP_RECONCILE_BINDING" != '{}' ] || {
fail 'NetworkPolicy observation has no baseline, tracked policies, or completed trigger'
return 1
}
while [ "$((SECONDS - started))" -le "$timeout" ]; do
[ "$(wrc_controller_snapshot)" = "$WRC_NP_OBSERVATION_PODS" ] || {
fail 'WRC observer changed; the NetworkPolicy window is incomplete'
return 1
}
logs="$(wrc_controller_logs "$WRC_NP_OBSERVATION_PODS" "$WRC_NP_OBSERVATION_SINCE")" || {
fail 'Unable to read the complete WRC observation window'
return 1
}
events="$(printf '%s' "$logs" | wrc_log_events "$WRC_NP_OBSERVATION_SINCE")" || return 1
# Match every final-apply and pre-DNS contraction event by namespace/name:
# contraction logs omit family, and a broken writer can mislabel it. Reject
# the contraction message regardless of action so a new action fails closed.
if printf '%s' "$events" | jq -e --argjson tracked "$WRC_NP_TRACKED" '
any(.[]; . as $event | any($tracked[]; .namespace == $event.namespace and .policy == $event.policy)
and (.msg == "contracted recipe network policy"
or .msg == "network policy read unavailable; retrying once"
or ((.msg // "") | test("^network policy (created|replaced|create failed|replace failed|read failed)$"))))' >/dev/null; then
fail 'NetworkPolicy window contains a write or failed operation, even if a no-op also occurred'
return 1
fi
while IFS= read -r row; do
current="$(kctl get networkpolicy "$(printf '%s' "$row" | jq -r '.policy')" \
-n "$(printf '%s' "$row" | jq -r '.namespace')" -o json)" || return 1
if ! printf '%s' "$current" | jq -e --argjson before "$row" '
.metadata.uid == $before.uid and .metadata.resourceVersion == $before.rv
and .metadata.deletionTimestamp == null' >/dev/null; then
fail 'NetworkPolicy changed since its pre-trigger baseline'
return 1
fi
done < <(printf '%s' "$WRC_NP_TRACKED" | jq -c '.[]')
missing="$(printf '%s' "$events" | jq -r --argjson tracked "$WRC_NP_TRACKED" \
--argjson binding "$WRC_NP_RECONCILE_BINDING" '
. as $events | ([ $tracked[] | . as $p | select(any($events[];
.policy == $p.policy and .namespace == $p.namespace and .family == $p.family and .msg == $p.message) | not)] | length)
+ (if any($events[]; .msg == "recipe reconciliation completed" and .recipe == $binding.recipe
and .namespace == $binding.namespace and .uid == $binding.uid and .generation == $binding.generation
and .phase == "active" and .requeueAfterMs == 0) then 0 else 1 end)')" || return 1
if [ "$missing" = 0 ] && [ "$((SECONDS - started))" -ge "$seconds" ]; then
ok 'Full NetworkPolicy window contains all no-op witnesses, no writes, and unchanged pre-trigger UID/resourceVersion'
WRC_NP_OBSERVATION_SINCE=""
return 0
fi
sleep "$POLL_INTERVAL"
done
fail 'NetworkPolicy observation lacked an exact completed reconcile or a required no-op witness'
return 1
}
# stdout carries a nonce-bound remote result; stderr/exec failure is a harness
# error. Only a completed connection timeout is a negative network observation.
wrc_http_probe() {
local namespace=$1 target=$2 host=$3 port=$4 seconds=${E2E_CONNECT_TIMEOUT:-6}
local nonce="np-$$-${RANDOM}-${RANDOM}" output begin end outcome encoded rest
[[ "$port" =~ ^[0-9]+$ ]] && [ "$port" -gt 0 ] && [ "$port" -le 65535 ] &&
[[ "$seconds" =~ ^[1-9][0-9]*$ ]] && [ "$seconds" -le 30 ] || return 1
# shellcheck disable=SC2016
if ! output="$(kctl exec "$target" -n "$namespace" -- sh -c '
export LC_ALL=C
for tool in nc grep base64 tr mktemp; do command -v "$tool" >/dev/null 2>&1 || exit 20; done
nonce=$1; host=$2; port=$3; seconds=$4
errors=$(mktemp) || exit 21
trap '\''rm -f "$errors"'\'' EXIT HUP INT TERM
printf "WRC_NP_BEGIN|%s\n" "$nonce"
# BusyBox nc is silent on connect timeout unless verbose mode is enabled.
# Keep stderr private and require a connection-specific diagnostic: neither
# a silent exit nor a timeout after connecting proves NetworkPolicy denial.
response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -v -w "$seconds" "$host" "$port" 2>"$errors")
status=$?
if [ "$status" -eq 0 ] && printf "%s" "$response" | grep -Eq "^HTTP/1\.[01] [2345][0-9][0-9]"; then
printf "WRC_NP_END|%s|http|" "$nonce"
printf "%s" "$response" | base64 | tr -d "\n"
printf "\n"
elif [ "$status" -eq 1 ] && [ -z "$response" ] && grep -Eqi "connect(ion)? .*timed out" "$errors"; then
printf "WRC_NP_END|%s|timeout|\n" "$nonce"
else
exit 22
fi
' -- "$nonce" "$host" "$port" "$seconds")"; then
fail "HTTP probe executor failed for ${namespace}/${target}; this is not NetworkPolicy denial" >&2
return 1
fi
begin="$(printf '%s\n' "$output" | sed -n '1p')"
end="$(printf '%s\n' "$output" | sed -n '2p')"
rest="$(printf '%s\n' "$output" | sed -n '3,$p')"
[ "$begin" = "WRC_NP_BEGIN|$nonce" ] && [ -z "$rest" ] &&
[[ "$end" == "WRC_NP_END|${nonce}|"* ]] || return 1
end="${end#WRC_NP_END|${nonce}|}"
outcome="${end%%|*}"
encoded="${end#*|}"
case "$outcome" in
http) [ -n "$encoded" ] || return 1 ;;
timeout) [ -z "$encoded" ] || return 1 ;;
*) return 1 ;;
esac
jq -cen --arg outcome "$outcome" --arg body "$encoded" '
{outcome:$outcome,body:$body}
| select((.outcome == "timeout" and .body == "") or
(.outcome == "http" and (.body | test("^[A-Za-z0-9+/]+={0,2}$"))
and ((.body | length) % 4 == 0)
and (.body | @base64d | test("^HTTP/1\\.[01] [2345][0-9][0-9]"))))'
}
wrc_assert_http_allowed() {
local description=$1 namespace=$2 target=$3 host=$4 port=$5 expected=${6:-} result
local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}
[[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1
while [ "$((SECONDS - started))" -lt "$timeout" ]; do
result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1
if printf '%s' "$result" | jq -e --arg expected "$expected" '
.outcome == "http" and ((.body | @base64d) | contains($expected))' >/dev/null; then
ok "$description"
return 0
fi
# Policy acknowledgement precedes CNI programming. Only valid remote
# network outcomes may be retried; executor failures abort immediately.
sleep "$POLL_INTERVAL"
done
fail "$description: expected an HTTP response from the reachable fixture"
return 1
}
wrc_assert_http_blocked() {
local description=$1 namespace=$2 target=$3 host=$4 port=$5 result
local started=$SECONDS timeout=${E2E_PROBE_TIMEOUT:-30}
[[ "$timeout" =~ ^[1-9][0-9]*$ ]] && [ "$timeout" -le 120 ] || return 1
while [ "$((SECONDS - started))" -lt "$timeout" ]; do
result="$(wrc_http_probe "$namespace" "$target" "$host" "$port")" || return 1
if printf '%s' "$result" | jq -e '.outcome == "timeout"' >/dev/null; then
ok "$description"
return 0
fi
sleep "$POLL_INTERVAL"
done
fail "$description: unexpected successful connection"
return 1
}
# A fixture-owned permission for one direction, peer and port. Complementary
# controls isolate the boundary being tested without changing platform policy.
wrc_create_connection_policy() {
local name=$1 direction=$2 namespace=$3 selector=$4 peer_namespace=$5 peer_selector=$6 port=$7
case "$direction" in Ingress|Egress) ;; *) return 1 ;; esac
jq -cn --arg name "$name" --arg direction "$direction" --arg namespace "$namespace" \
--argjson selector "$selector" --arg peerNamespace "$peer_namespace" \
--argjson peerSelector "$peer_selector" --argjson port "$port" '
{namespaceSelector:{matchLabels:{"kubernetes.io/metadata.name":$peerNamespace}},
podSelector:{matchLabels:$peerSelector}} as $peer
| {ports:[{protocol:"TCP",port:$port}]} as $ports
| {apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",
metadata:{name:$name,namespace:$namespace},
spec:({podSelector:{matchLabels:$selector},policyTypes:[$direction]} +
if $direction == "Ingress" then {ingress:[($ports + {from:[$peer]})]}
else {egress:[($ports + {to:[$peer]})]} end)}' | wrc_create_owned
}