Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(e2e): preserve probe and cleanup evidence boundaries
Alfredo Lopez56baecf84ff3
diff --git a/Makefile b/Makefileindex 3f30839ec..872bf2967 100644--- a/Makefile+++ b/Makefile@@ -1108,6 +1108,7 @@ test-wrc-networkpolicy-contracts: ## Run hermetic WRC NetworkPolicy runner, probbash scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.shbash scripts/tests/test-wrc-networkpolicy-probe-contract.shbash scripts/tests/test-wrc-fixtures.sh+ bash scripts/tests/test-wrc-fixtures-parent-ownership.sh.PHONY: test-e2e-codex-subscription-network-boundarytest-e2e-codex-subscription-network-boundary: ## Codex LLM proxy NetworkPolicy boundary (exit 3 before deploy)diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.mdindex 80273e931..5fe6d798d 100644--- a/docs/deploy/workflow-recipes-guide.md+++ b/docs/deploy/workflow-recipes-guide.md@@ -388,7 +388,9 @@ Negative controls first prove reachability from the same probe with a narrow tempermission, remove only that permission, observe a remotely executed connection timeout,and prove reachability again. Complementary permissions isolate ingress from egress.An exec, authorization, missing-tool or malformed-response error fails the test instead-of counting as a policy denial. Condition-based polling accommodates CNI propagation.+of counting as a policy denial. Silent failures and read timeouts after a connection+opens are also errors; only a connection-timeout diagnostic counts as denial.+Condition-based polling accommodates CNI propagation.Every no-churn observation begins before the parent trigger. It requires the exactUID/generation completion receipt, every policy/family no-op witness, unchanged policy@@ -398,7 +400,10 @@ resourceVersion does not change. Exact API-call counts are asserted separately itests; the runtime logs are not a substitute for exhaustive Kubernetes audit accounting.Fixtures use unique run IDs, collision-safe creation, UID-preconditioned deletion and-owned dynamic port-forwards. Cleanup never enrolls an existing sample by name, and a+owned dynamic port-forwards. Generated children are enrolled only while the same owned+parent UID is verified before and after discovery; parent absence permits cleanup of+previously recorded child UIDs only. A failed parent deletion stops child cleanup.+Cleanup never enrolls an existing sample by name, and acleanup failure fails the suite. `E2E_KEEP_RESOURCES=1` explicitly retains runresources for inspection, while still stopping owned port-forwards.diff --git a/scripts/e2e/_lib/wrc-fixtures.sh b/scripts/e2e/_lib/wrc-fixtures.shindex 36724e6ac..da65fed5d 100755--- a/scripts/e2e/_lib/wrc-fixtures.sh+++ b/scripts/e2e/_lib/wrc-fixtures.sh@@ -76,37 +76,69 @@ wrc_delete_owned() {}wrc_capture_recipe_children() {- local ns=$1 name=$2 uid=$3 target objects+ local ns=$1 name=$2 uid=$3 target objects parent owned candidates='[]' selected+ owned="$(jq -sc --arg ns "$ns" --arg name "$name" --arg uid "$uid" '+ [.[] | select(.kind=="WorkflowRecipe" and .metadata.namespace==$ns+ and .metadata.name==$name and .metadata.uid==$uid)] | last // empty' "$WRC_FIXTURE_LEDGER")" || return 1+ [[ -n "$owned" ]] || return 1+ parent="$(kctl get workflowrecipe "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1+ # An absent parent cannot authorize new label-based discovery. Its previously+ # recorded children can still be deleted by their immutable UID on a retry.+ [[ -n "$parent" ]] || return 0+ jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$parent" >/dev/null || {+ echo "Recipe ownership changed before child capture: $ns/$name" >&2; return 1;+ }for target in "$ns" sandbox-ui; doobjects="$(kctl get deployment,service,configmap,networkpolicy -n "$target" --request-timeout=30s -o json)" || return 1# Cross-namespace UI resources cannot have recipe ownerReferences. WRC's# complete recipe namespace/name labels bind those to our collision-free CR.- jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '.items[] |+ selected="$(jq -c --arg uid "$uid" --arg ns "$ns" --arg name "$name" '[.items[] |select(any(.metadata.ownerReferences[]?; .uid==$uid) or(.metadata.labels["clerum.io/recipe-namespace"]==$ns and.metadata.labels["clerum.io/recipe-name"]==$name) or(.metadata.namespace==$ns and.metadata.labels["clerum.io/managed-by"]=="workflow-recipes" and- .metadata.labels["clerum.io/recipe"]==$name))' <<< "$objects" |- while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done || return 1+ .metadata.labels["clerum.io/recipe"]==$name))+ | {apiVersion,kind,metadata:{name:.metadata.name,namespace:.metadata.namespace,+ uid:.metadata.uid,labels:(.metadata.labels // {})}}]' <<< "$objects")" || return 1+ candidates="$(jq -cn --argjson previous "$candidates" --argjson selected "$selected" '$previous + $selected')" || return 1done+ # Keep discovery provisional until the same parent is observed after the+ # inventory. Otherwise a replacement recipe could lend us its children's UIDs.+ parent="$(kctl get workflowrecipe "$name" -n "$ns" --ignore-not-found --request-timeout=30s -o json)" || return 1+ [[ -n "$parent" ]] && jq -e --argjson owned "$owned" '.metadata.uid==$owned.metadata.uid and+ ((.metadata.labels // {}) | contains($owned.metadata.labels))' <<< "$parent" >/dev/null || {+ echo "Recipe ownership changed during child capture: $ns/$name" >&2; return 1;+ }+ jq -c '.[]' <<< "$candidates" |+ while IFS= read -r object; do printf '%s\n' "$object" | wrc_record_owned || return 1; done}wrc_cleanup_owned() {- local status=0 ns kind name uid+ local status=0 ns kind name uid parents records+ # Parse before process substitutions, so a truncated/corrupt ledger cannot+ # look like an empty successful cleanup.+ parents="$(jq -sc '[.[] | select(.kind=="WorkflowRecipe")]+ | unique_by([.metadata.namespace,.metadata.name])' "$WRC_FIXTURE_LEDGER")" || return 1# Capture generated resources before deleting recipes, including the UI# namespace where Kubernetes cannot garbage-collect a cross-namespace owner.while IFS=$'\t' read -r ns name uid; do[[ -n "$name" ]] || continue- wrc_capture_recipe_children "$ns" "$name" "$uid" || status=1- done < <(jq -r 'select(.kind=="WorkflowRecipe") | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' "$WRC_FIXTURE_LEDGER")- # Recipes first stops reconciliation from recreating children during cleanup.+ wrc_capture_recipe_children "$ns" "$name" "$uid" || return 1+ done < <(jq -r '.[] | [.metadata.namespace,.metadata.name,.metadata.uid]|@tsv' <<< "$parents")+ # Every parent must be removed before any child cleanup. A failed parent+ # ownership check or deletion must not be followed by destructive side effects.+ while IFS=$'\t' read -r ns name; do+ [[ -n "$name" ]] || continue+ wrc_delete_owned "$ns" WorkflowRecipe "$name" || return 1+ done < <(jq -r '.[] | [.metadata.namespace,.metadata.name]|@tsv' <<< "$parents")+ records="$(jq -sc 'map(select(.kind!="WorkflowRecipe"))+ | unique_by([.kind,.metadata.namespace,.metadata.name])' "$WRC_FIXTURE_LEDGER")" || return 1while IFS=$'\t' read -r ns kind name; do[[ -n "$name" ]] || continuewrc_delete_owned "$ns" "$kind" "$name" || status=1- done < <(jq -sr 'unique_by([.kind,.metadata.namespace,.metadata.name]) |- sort_by(if .kind=="WorkflowRecipe" then 0 else 1 end)[] |- [.metadata.namespace,.kind,.metadata.name]|@tsv' "$WRC_FIXTURE_LEDGER")+ done < <(jq -r '.[] | [.metadata.namespace,.kind,.metadata.name]|@tsv' <<< "$records")return "$status"}diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.shindex 1d9a78336..dff36bd82 100755--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh@@ -293,13 +293,16 @@ wrc_http_probe() {errors=$(mktemp) || exit 21trap '\''rm -f "$errors"'\'' EXIT HUP INT TERMprintf "WRC_NP_BEGIN|%s\n" "$nonce"- response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -w "$seconds" "$host" "$port" 2>"$errors")+ # BusyBox nc is silent on connect timeout unless verbose mode is enabled.+ # Keep stderr private and require a connection-specific diagnostic: neither+ # a silent exit nor a timeout after connecting proves NetworkPolicy denial.+ response=$(printf "GET / HTTP/1.0\r\nHost: e2e\r\nConnection: close\r\n\r\n" | nc -v -w "$seconds" "$host" "$port" 2>"$errors")status=$?if [ "$status" -eq 0 ] && printf "%s" "$response" | grep -Eq "^HTTP/1\.[01] [2345][0-9][0-9]"; thenprintf "WRC_NP_END|%s|http|" "$nonce"printf "%s" "$response" | base64 | tr -d "\n"printf "\n"- elif [ "$status" -ne 0 ] && [ -z "$response" ] && grep -Eqi "timed out|timeout" "$errors"; then+ elif [ "$status" -eq 1 ] && [ -z "$response" ] && grep -Eqi "connect(ion)? .*timed out" "$errors"; thenprintf "WRC_NP_END|%s|timeout|\n" "$nonce"elseexit 22diff --git a/scripts/tests/test-wrc-fixtures-parent-ownership.sh b/scripts/tests/test-wrc-fixtures-parent-ownership.shnew file mode 100755index 000000000..205d6d2ac--- /dev/null+++ b/scripts/tests/test-wrc-fixtures-parent-ownership.sh@@ -0,0 +1,87 @@+#!/usr/bin/env bash+# Exercise the real cleanup: API transport alone models parent recreation and+# deletion failure. A nonzero cleanup status must never hide a foreign delete.+# shellcheck disable=SC2329+set -euo pipefail+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+CASE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wrc-fixture-parents.XXXXXX")"+trap 'rm -rf -- "$CASE_ROOT"' EXIT++run_case() (+ local scenario=$1 status=0 expected_status=1 expected_deleted=0 records+ # shellcheck source=../e2e/_lib/wrc-fixtures.sh+ source "$ROOT/scripts/e2e/_lib/wrc-fixtures.sh"+ WRC_FIXTURE_DIR="$CASE_ROOT/$scenario"+ mkdir "$WRC_FIXTURE_DIR"+ WRC_FIXTURE_LEDGER="$WRC_FIXTURE_DIR/owned.jsonl"+ local fixture_parent='{"apiVersion":"clerum.io/v1alpha1","kind":"WorkflowRecipe","metadata":{"name":"fixture","namespace":"sandbox-recipes","uid":"old-parent","labels":{"e2e.clerum.io/run":"oldrun"}}}'+ local fixture_child='{"apiVersion":"v1","kind":"Service","metadata":{"name":"child","namespace":"sandbox-ui","uid":"child-uid","labels":{"clerum.io/recipe-namespace":"sandbox-recipes","clerum.io/recipe-name":"fixture"}}}'+ printf '%s\n' "$fixture_parent" > "$WRC_FIXTURE_LEDGER"+ if [ "$scenario" = corrupt-ledger ]; then printf '{' > "$WRC_FIXTURE_LEDGER"; fi+ if [ "$scenario" = absent-recorded ]; then printf '%s\n' "$fixture_child" | wrc_record_owned; fi+ kctl() {+ case "$1:$2" in+ get:workflowrecipe|get:WorkflowRecipe)+ [ "$scenario" != read-error ] || return 1+ if [[ "$scenario" == absent* ]] || [ -f "$WRC_FIXTURE_DIR/parent-gone" ]; then return 0; fi+ if [ "$scenario" = replaced ] || [ -f "$WRC_FIXTURE_DIR/swapped" ]; then+ printf '%s' "$fixture_parent" | jq '.metadata.uid="new-parent" | .metadata.labels["e2e.clerum.io/run"]="newrun"'+ else+ printf '%s\n' "$fixture_parent"+ fi ;;+ get:deployment,service,configmap,networkpolicy)+ if [ "$scenario" = swap-during-list ]; then : > "$WRC_FIXTURE_DIR/swapped"; fi+ if [ "$4" = sandbox-ui ]; then jq -cn --argjson child "$fixture_child" '{items:[$child]}'+ else printf '{"items":[]}\n'; fi ;;+ get:Service)+ [ ! -f "$WRC_FIXTURE_DIR/child-deleted" ] || return 0+ printf '%s\n' "$fixture_child" ;;+ delete:--raw)+ case "$3" in+ /apis/clerum.io/v1alpha1/namespaces/sandbox-recipes/workflowrecipes/fixture)+ [ "$scenario" != parent-delete-fails ] || return 1+ jq -e '.preconditions.uid=="old-parent"' "$6" >/dev/null || return 1+ : > "$WRC_FIXTURE_DIR/parent-gone" ;;+ /api/v1/namespaces/sandbox-ui/services/child)+ jq -e '.preconditions.uid=="child-uid"' "$6" >/dev/null || return 1+ : > "$WRC_FIXTURE_DIR/child-deleted" ;;+ *) return 1 ;;+ esac ;;+ wait:*) return 0 ;;+ *) return 1 ;;+ esac+ }+ wrc_cleanup_owned || status=$?+ case "$scenario" in+ healthy|absent-recorded) expected_status=0; expected_deleted=1 ;;+ absent) expected_status=0 ;;+ esac+ local deleted=0+ [ ! -f "$WRC_FIXTURE_DIR/child-deleted" ] || deleted=1+ [ "$status" -eq "$expected_status" ] && [ "$deleted" -eq "$expected_deleted" ] || {+ printf 'FAIL: %s cleanup=%s child_deleted=%s\n' "$scenario" "$status" "$deleted" >&2+ return 1+ }+ case "$scenario" in+ replaced|swap-during-list|read-error|absent)+ records="$(jq -s 'length' "$WRC_FIXTURE_LEDGER")"+ [ "$records" -eq 1 ] || { printf 'FAIL: %s polluted the ownership ledger\n' "$scenario" >&2; return 1; }+ ;;+ absent-recorded)+ [ "$(jq -s 'length' "$WRC_FIXTURE_LEDGER")" -eq 2 ] || return 1+ ;;+ corrupt-ledger)+ [ "$(cat "$WRC_FIXTURE_LEDGER")" = '{' ] || return 1+ ;;+ esac+)++failed=0+for scenario in healthy replaced swap-during-list read-error absent absent-recorded parent-delete-fails corrupt-ledger; do+ if run_case "$scenario"; then+ printf 'PASS: cleanup parent boundary %s\n' "$scenario"+ else+ failed=1+ fi+done+exit "$failed"diff --git a/scripts/tests/test-wrc-fixtures.sh b/scripts/tests/test-wrc-fixtures.shindex 7fd15cded..d05193bfa 100755--- a/scripts/tests/test-wrc-fixtures.sh+++ b/scripts/tests/test-wrc-fixtures.sh@@ -28,6 +28,10 @@ kctl() {;;get)[[ "$MODE" != read-error ]] || return 1+ if [[ "$2" == workflowrecipe ]]; then+ cat "$TEST_ROOT/parent.json"+ return+ fiif [[ "$2" == deployment,service,configmap,networkpolicy ]]; thencat "$TEST_ROOT/children.json"return@@ -129,8 +133,10 @@ jq -n '{items:[{apiVersion:"networking.k8s.io/v1",kind:"NetworkPolicy",metadata:{name:"foreign",namespace:"sandbox-recipes",uid:"foreign1",labels:{"clerum.io/managed-by":"workflow-recipes","clerum.io/recipe":"another"}}},{apiVersion:"v1",kind:"Service",metadata:{name:"foreign-ns",namespace:"sandbox-ui",uid:"foreign2",labels:{"clerum.io/recipe-namespace":"other-namespace","clerum.io/recipe-name":"fixture"}}}]}' > "$TEST_ROOT/children.json"+jq -n '{apiVersion:"clerum.io/v1alpha1",kind:"WorkflowRecipe",metadata:{name:"fixture",namespace:"sandbox-recipes",uid:"recipe-uid",labels:{"e2e.clerum.io/run":"contract123"}}}' > "$TEST_ROOT/parent.json"+wrc_record_owned < "$TEST_ROOT/parent.json"wrc_capture_recipe_children sandbox-recipes fixture recipe-uid-jq -se 'map(.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/null+jq -se 'map(select(.kind!="WorkflowRecipe")|.metadata.name)|unique|sort == ["cross-ns","owned","ownerless"]' "$WRC_FIXTURE_LEDGER" >/dev/nullecho 'PASS: generated ownerless/cross-namespace children captured without foreign resources'# Validate rejected IDs before any fixture directory or cluster operation.diff --git a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh b/scripts/tests/test-wrc-networkpolicy-probe-contract.shindex 76e9f9d8c..7883ba4db 100755--- a/scripts/tests/test-wrc-networkpolicy-probe-contract.sh+++ b/scripts/tests/test-wrc-networkpolicy-probe-contract.sh@@ -67,7 +67,13 @@ esaccase "$PROBE_SCENARIO" inhttp) printf 'HTTP/1.0 200 OK\r\n\r\nfixture-business-signal' ;;http-403) printf 'HTTP/1.0 403 Forbidden\r\n\r\nfixture-auth-denied' ;;- timeout) printf 'nc: connect timed out\n' >&2; exit 1 ;;+ # BusyBox 1.36.1 was observed to time out silently without -v. Exercise the+ # real CLI seam: a fabricated timeout message used to hide this runtime bug.+ timeout)+ case " $* " in *' -v '*) printf 'nc: fixture (fixture:8080): Connection timed out\n' >&2 ;; esac+ exit 1 ;;+ silent-error) exit 1 ;;+ read-timeout) printf 'nc: fixture (fixture:8080) open\nnc: net timeout\n' >&2; exit 1 ;;refused) printf 'nc: Connection refused\n' >&2; exit 1 ;;empty) exit 0 ;;malformed) printf 'not an HTTP response' ;;@@ -153,7 +159,7 @@ expect_reject 'HTTP authorization rejection as NetworkPolicy denial' probe_blockPROBE_SCENARIO=timeoutexpect_pass 'completed remote connection timeout as a network observation' probe_blockedexpect_reject 'timeout as an allowed route' probe_allowed-for PROBE_SCENARIO in empty malformed refused; do+for PROBE_SCENARIO in empty malformed refused silent-error read-timeout; doexpect_reject "remote $PROBE_SCENARIO response as NetworkPolicy denial" probe_blockeddonePROBE_SCENARIO=http