Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(e2e): preserve probe and cleanup evidence boundaries

Alfredo Lopez56baecf84ff3
diff --git a/Makefile b/Makefile
index 3f30839ec..872bf2967 100644
--- a/Makefile
+++ b/Makefile
@@ -1108,6 +1108,7 @@ test-wrc-networkpolicy-contracts: ## Run hermetic WRC NetworkPolicy runner, prob
bash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
bash scripts/tests/test-wrc-networkpolicy-probe-contract.sh
bash scripts/tests/test-wrc-fixtures.sh
+ bash scripts/tests/test-wrc-fixtures-parent-ownership.sh
.PHONY: test-e2e-codex-subscription-network-boundary
test-e2e-codex-subscription-network-boundary: ## Codex LLM proxy NetworkPolicy boundary (exit 3 before deploy)
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 80273e931..5fe6d798d 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -388,7 +388,9 @@ Negative controls first prove reachability from the same probe with a narrow tem
permission, remove only that permission, observe a remotely executed connection timeout,
and prove reachability again. Complementary permissions isolate ingress from egress.
An exec, authorization, missing-tool or malformed-response error fails the test instead
-of counting as a policy denial. Condition-based polling accommodates CNI propagation.
+of counting as a policy denial. Silent failures and read timeouts after a connection
+opens are also errors; only a connection-timeout diagnostic counts as denial.
+Condition-based polling accommodates CNI propagation.
Every no-churn observation begins before the parent trigger. It requires the exact
UID/generation completion receipt, every policy/family no-op witness, unchanged policy
@@ -398,7 +400,10 @@ resourceVersion does not change. Exact API-call counts are asserted separately i
tests; the runtime logs are not a substitute for exhaustive Kubernetes audit accounting.
Fixtures use unique run IDs, collision-safe creation, UID-preconditioned deletion and
-owned dynamic port-forwards. Cleanup never enrolls an existing sample by name, and a
+owned dynamic port-forwards. Generated children are enrolled only while the same owned
+parent UID is verified before and after discovery; parent absence permits cleanup of
+previously recorded child UIDs only. A failed parent deletion stops child cleanup.
+Cleanup never enrolls an existing sample by name, and a
cleanup failure fails the suite. `E2E_KEEP_RESOURCES=1` explicitly retains run
resources for inspection, while still stopping owned port-forwards.
diff --git a/scripts/e2e/_lib/wrc-fixtures.sh b/scripts/e2e/_lib/wrc-fixtures.sh
index 36724e6ac..da65fed5d 100755
--- a/scripts/e2e/_lib/wrc-fixtures.sh
+++ b/scripts/e2e/_lib/wrc-fixtures.sh
@@ -76,37 +76,69 @@ wrc_delete_owned() {
}
wrc_capture_recipe_children() {
- local ns=$1 name=$2 uid=$3 target objects
+ local ns=$1 name=$2 uid=$3 target objects parent owned candidates='[]' selected
+ owned="$(jq -sc --arg ns "$ns" --arg name "$name" --arg uid "$uid" '
+ [.[] | select(.kind=="WorkflowRecipe" and .metadata.namespace==$ns
+ and .metadata.name==$name and .metadata.uid==$uid)] | last // empty' "$WRC_FIXTURE_LEDGER")" || return 1
+ [[ -n "$owned" ]] || return 1
+ parent="$(kctl get workflowrecipe "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1
+ # An absent parent cannot authorize new label-based discovery. Its previously
+ # recorded children can still be deleted by their immutable UID on a retry.
+ [[ -n "$parent" ]] || return 0
+ jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and
+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$parent" >/dev/null || {
+ echo "Recipe ownership changed before child capture: $ns/$name" >&2; return 1;
+ }
for target in "$ns" sandbox-ui; do
objects="$(kctl get deployment,service,configmap,networkpolicy -n "$target" --request-timeout=30s -o json)" || return 1
# Cross-namespace UI resources cannot have recipe ownerReferences. WRC's
# complete recipe namespace/name labels bind those to our collision-free CR.
- jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '.items[] |
+ selected="$(jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '[.items[] |
select(any(.metadata.ownerReferences[]?; .uid==$uid) or
(.metadata.labels["clerum.io/recipe-namespace"]==$ns and
.metadata.labels["clerum.io/recipe-name"]==$name) or
(.metadata.namespace==$ns and
.metadata.labels["clerum.io/managed-by"]=="workflow-recipes" and
- .metadata.labels["clerum.io/recipe"]==$name))' <<< "$objects" |
- while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done || return 1
+ .metadata.labels["clerum.io/recipe"]==$name))
+ | {apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace,
+ uid:.metadata.uid,labels:(.metadata.labels // {})}}]' <<< "$objects")" || return 1
+ candidates="$(jq -cn --argjson previous "$candidates" --argjson selected "$selected" '$previous + $selected')" || return 1
done
+ # Keep discovery provisional until the same parent is observed after the
+ # inventory. Otherwise a replacement recipe could lend us its children's UIDs.
+ parent="$(kctl get workflowrecipe "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1
+ [[ -n "$parent" ]] && jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and
+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$parent" >/dev/null || {
+ echo "Recipe ownership changed during child capture: $ns/$name" >&2; return 1;
+ }
+ jq -c '.[]' <<< "$candidates" |
+ while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done
}
wrc_cleanup_owned() {
- local status=0 ns kind name uid
+ local status=0 ns kind name uid parents records
+ # Parse before process substitutions, so a truncated/corrupt ledger cannot
+ # look like an empty successful cleanup.
+ parents="$(jq -sc '[.[] | select(.kind=="WorkflowRecipe")]
+ | unique_by([.metadata.namespace,.metadata.name])' "$WRC_FIXTURE_LEDGER")" || return 1
# Capture generated resources before deleting recipes, including the UI
# namespace where Kubernetes cannot garbage-collect a cross-namespace owner.
while IFS=$'\t' read -r ns name uid; do
[[ -n "$name" ]] || continue
- wrc_capture_recipe_children "$ns" "$name" "$uid" || status=1
- done < <(jq -r 'select(.kind=="WorkflowRecipe") | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' "$WRC_FIXTURE_LEDGER")
- # Recipes first stops reconciliation from recreating children during cleanup.
+ wrc_capture_recipe_children "$ns" "$name" "$uid" || return 1
+ done < <(jq -r '.[] | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' <<< "$parents")
+ # Every parent must be removed before any child cleanup. A failed parent
+ # ownership check or deletion must not be followed by destructive side effects.
+ while IFS=$'\t' read -r ns name; do
+ [[ -n "$name" ]] || continue
+ wrc_delete_owned "$ns" WorkflowRecipe "$name" || return 1
+ done < <(jq -r '.[] | [.metadata.namespace,.metadata.name]|@tsv' <<< "$parents")
+ records="$(jq -sc 'map(select(.kind!="WorkflowRecipe"))
+ | unique_by([.kind,.metadata.namespace,.metadata.name])' "$WRC_FIXTURE_LEDGER")" || return 1
while IFS=$'\t' read -r ns kind name; do
[[ -n "$name" ]] || continue
wrc_delete_owned "$ns" "$kind" "$name" || status=1
- done < <(jq -sr 'unique_by([.kind,.metadata.namespace,.metadata.name]) |
- sort_by(if .kind=="WorkflowRecipe" then 0 else 1 end)[] |
- [.metadata.namespace,.kind,.metadata.name]|@tsv' "$WRC_FIXTURE_LEDGER")
+ done < <(jq -r '.[] | [.metadata.namespace,.kind,.metadata.name]|@tsv' <<< "$records")
return "$status"
}
diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
index 1d9a78336..dff36bd82 100755
--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
@@ -293,13 +293,16 @@ wrc_http_probe() {
errors=$(mktemp) || exit 21
trap '\''rm -f "$errors"'\'' EXIT HUP INT TERM
printf "WRC_NP_BEGIN|%s\n" "$nonce"
- response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$seconds" "$host" "$port" 2>"$errors")
+ # BusyBox nc is silent on connect timeout unless verbose mode is enabled.
+ # Keep stderr private and require a connection-specific diagnostic: neither
+ # a silent exit nor a timeout after connecting proves NetworkPolicy denial.
+ response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -v -w "$seconds" "$host" "$port" 2>"$errors")
status=$?
if [ "$status" -eq 0 ] && printf "%s" "$response" | grep -Eq "^HTTP/1\.[01] [2345][0-9][0-9]"; then
printf "WRC_NP_END|%s|http|" "$nonce"
printf "%s" "$response" | base64 | tr -d "\n"
printf "\n"
- elif [ "$status" -ne 0 ] && [ -z "$response" ] && grep -Eqi "timed out|timeout" "$errors"; then
+ elif [ "$status" -eq 1 ] && [ -z "$response" ] && grep -Eqi "connect(ion)? .*timed out" "$errors"; then
printf "WRC_NP_END|%s|timeout|\n" "$nonce"
else
exit 22
diff --git a/scripts/tests/test-wrc-fixtures-parent-ownership.sh b/scripts/tests/test-wrc-fixtures-parent-ownership.sh
new file mode 100755
index 000000000..205d6d2ac
--- /dev/null
+++ b/scripts/tests/test-wrc-fixtures-parent-ownership.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Exercise the real cleanup: API transport alone models parent recreation and
+# deletion failure. A nonzero cleanup status must never hide a foreign delete.
+# shellcheck disable=SC2329
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CASE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-fixture-parents.XXXXXX")"
+trap 'rm -rf -- "$CASE_ROOT"' EXIT
+
+run_case() (
+ local scenario=$1 status=0 expected_status=1 expected_deleted=0 records
+ # shellcheck source=../e2e/_lib/wrc-fixtures.sh
+ source "$ROOT/scripts/e2e/_lib/wrc-fixtures.sh"
+ WRC_FIXTURE_DIR="$CASE_ROOT/$scenario"
+ mkdir "$WRC_FIXTURE_DIR"
+ WRC_FIXTURE_LEDGER="$WRC_FIXTURE_DIR/owned.jsonl"
+ local fixture_parent='{"apiVersion":"clerum.io/v1alpha1","kind":"WorkflowRecipe","metadata":{"name":"fixture","namespace":"sandbox-recipes","uid":"old-parent","labels":{"e2e.clerum.io/run":"oldrun"}}}'
+ local fixture_child='{"apiVersion":"v1","kind":"Service","metadata":{"name":"child","namespace":"sandbox-ui","uid":"child-uid","labels":{"clerum.io/recipe-namespace":"sandbox-recipes","clerum.io/recipe-name":"fixture"}}}'
+ printf '%s\n' "$fixture_parent" > "$WRC_FIXTURE_LEDGER"
+ if [ "$scenario" = corrupt-ledger ]; then printf '{' > "$WRC_FIXTURE_LEDGER"; fi
+ if [ "$scenario" = absent-recorded ]; then printf '%s\n' "$fixture_child" | wrc_record_owned; fi
+ kctl() {
+ case "$1:$2" in
+ get:workflowrecipe|get:WorkflowRecipe)
+ [ "$scenario" != read-error ] || return 1
+ if [[ "$scenario" == absent* ]] || [ -f "$WRC_FIXTURE_DIR/parent-gone" ]; then return 0; fi
+ if [ "$scenario" = replaced ] || [ -f "$WRC_FIXTURE_DIR/swapped" ]; then
+ printf '%s' "$fixture_parent" | jq '.metadata.uid="new-parent" | .metadata.labels["e2e.clerum.io/run"]="newrun"'
+ else
+ printf '%s\n' "$fixture_parent"
+ fi ;;
+ get:deployment,service,configmap,networkpolicy)
+ if [ "$scenario" = swap-during-list ]; then : > "$WRC_FIXTURE_DIR/swapped"; fi
+ if [ "$4" = sandbox-ui ]; then jq -cn --argjson child "$fixture_child" '{items:[$child]}'
+ else printf '{"items":[]}\n'; fi ;;
+ get:Service)
+ [ ! -f "$WRC_FIXTURE_DIR/child-deleted" ] || return 0
+ printf '%s\n' "$fixture_child" ;;
+ delete:--raw)
+ case "$3" in
+ /apis/clerum.io/v1alpha1/namespaces/sandbox-recipes/workflowrecipes/fixture)
+ [ "$scenario" != parent-delete-fails ] || return 1
+ jq -e '.preconditions.uid=="old-parent"' "$6" >/dev/null || return 1
+ : > "$WRC_FIXTURE_DIR/parent-gone" ;;
+ /api/v1/namespaces/sandbox-ui/services/child)
+ jq -e '.preconditions.uid=="child-uid"' "$6" >/dev/null || return 1
+ : > "$WRC_FIXTURE_DIR/child-deleted" ;;
+ *) return 1 ;;
+ esac ;;
+ wait:*) return 0 ;;
+ *) return 1 ;;
+ esac
+ }
+ wrc_cleanup_owned || status=$?
+ case "$scenario" in
+ healthy|absent-recorded) expected_status=0; expected_deleted=1 ;;
+ absent) expected_status=0 ;;
+ esac
+ local deleted=0
+ [ ! -f "$WRC_FIXTURE_DIR/child-deleted" ] || deleted=1
+ [ "$status" -eq "$expected_status" ] && [ "$deleted" -eq "$expected_deleted" ] || {
+ printf 'FAIL: %s cleanup=%s child_deleted=%s\n' "$scenario" "$status" "$deleted" >&2
+ return 1
+ }
+ case "$scenario" in
+ replaced|swap-during-list|read-error|absent)
+ records="$(jq -s 'length' "$WRC_FIXTURE_LEDGER")"
+ [ "$records" -eq 1 ] || { printf 'FAIL: %s polluted the ownership ledger\n' "$scenario" >&2; return 1; }
+ ;;
+ absent-recorded)
+ [ "$(jq -s 'length' "$WRC_FIXTURE_LEDGER")" -eq 2 ] || return 1
+ ;;
+ corrupt-ledger)
+ [ "$(cat "$WRC_FIXTURE_LEDGER")" = '{' ] || return 1
+ ;;
+ esac
+)
+
+failed=0
+for scenario in healthy replaced swap-during-list read-error absent absent-recorded parent-delete-fails corrupt-ledger; do
+ if run_case "$scenario"; then
+ printf 'PASS: cleanup parent boundary %s\n' "$scenario"
+ else
+ failed=1
+ fi
+done
+exit "$failed"
diff --git a/scripts/tests/test-wrc-fixtures.sh b/scripts/tests/test-wrc-fixtures.sh
index 7fd15cded..d05193bfa 100755
--- a/scripts/tests/test-wrc-fixtures.sh
+++ b/scripts/tests/test-wrc-fixtures.sh
@@ -28,6 +28,10 @@ kctl() {
;;
get)
[[ "$MODE" != read-error ]] || return 1
+ if [[ "$2" == workflowrecipe ]]; then
+ cat "$TEST_ROOT/parent.json"
+ return
+ fi
if [[ "$2" == deployment,service,configmap,networkpolicy ]]; then
cat "$TEST_ROOT/children.json"
return
@@ -129,8 +133,10 @@ jq -n '{items:[
{apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"foreign",namespace:"sandbox-recipes",uid:"foreign1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"another"}}},
{apiVersion:"v1",kind:"Service",metadata:{name:"foreign-ns",namespace:"sandbox-ui",uid:"foreign2",labels:{"clerum.io/recipe-namespace":"other-namespace","clerum.io/recipe-name":"fixture"}}}
]}' > "$TEST_ROOT/children.json"
+jq -n '{apiVersion:"clerum.io/v1alpha1",kind:"WorkflowRecipe",metadata:{name:"fixture",namespace:"sandbox-recipes",uid:"recipe-uid",labels:{"e2e.clerum.io/run":"contract123"}}}' > "$TEST_ROOT/parent.json"
+wrc_record_owned < "$TEST_ROOT/parent.json"
wrc_capture_recipe_children sandbox-recipes fixture recipe-uid
-jq -se 'map(.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/null
+jq -se 'map(select(.kind!="WorkflowRecipe")|.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/null
echo 'PASS: generated ownerless/cross-namespace children captured without foreign resources'
# Validate rejected IDs before any fixture directory or cluster operation.
diff --git a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh
index 76e9f9d8c..7883ba4db 100755
--- a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh
+++ b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh
@@ -67,7 +67,13 @@ esac
case "$PROBE_SCENARIO" in
http) printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal' ;;
http-403) printf 'HTTP/1.0 403 Forbidden\r\n\r\nfixture-auth-denied' ;;
- timeout) printf 'nc: connect timed out\n' >&2; exit 1 ;;
+ # BusyBox 1.36.1 was observed to time out silently without -v. Exercise the
+ # real CLI seam: a fabricated timeout message used to hide this runtime bug.
+ timeout)
+ case " $* " in *' -v '*) printf 'nc: fixture (fixture:8080): Connection timed out\n' >&2 ;; esac
+ exit 1 ;;
+ silent-error) exit 1 ;;
+ read-timeout) printf 'nc: fixture (fixture:8080) open\nnc: net timeout\n' >&2; exit 1 ;;
refused) printf 'nc: Connection refused\n' >&2; exit 1 ;;
empty) exit 0 ;;
malformed) printf 'not an HTTP response' ;;
@@ -153,7 +159,7 @@ expect_reject 'HTTP authorization rejection as NetworkPolicy denial' probe_block
PROBE_SCENARIO=timeout
expect_pass 'completed remote connection timeout as a network observation' probe_blocked
expect_reject 'timeout as an allowed route' probe_allowed
-for PROBE_SCENARIO in empty malformed refused; do
+for PROBE_SCENARIO in empty malformed refused silent-error read-timeout; do
expect_reject "remote $PROBE_SCENARIO response as NetworkPolicy denial" probe_blocked
done
PROBE_SCENARIO=http