Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(wrc): requeue transient NetworkPolicy races
Alfredo Lopez8968b58499fd
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.mdindex 3462f707a..0a123f28d 100644--- a/docs/deploy/workflow-recipes-guide.md+++ b/docs/deploy/workflow-recipes-guide.md@@ -334,6 +334,10 @@ it cannot hide later drift.owner forces one optimistic-concurrency replace using the `resourceVersion` from thesame validated read on the next parent recipe reconcile. NetworkPolicy-only changes donot emit a WorkflowRecipe event. The following reconcile is read-only.+- **Retryable races schedule their own recovery.** A terminating object, create conflict+ whose winner disappears, missing live `resourceVersion`, replace 404/409 or transient+ API failure re-enqueues the parent with bounded backoff. A status-only update is not+ treated as the retry trigger.- **Other spec-hash-managed objects retain their existing drift contract.** For example,a manually scaled recipe Deployment is not reverted until the desired manifest changes;its replica-health drift remains visible in recipe status.@@ -360,7 +364,9 @@ NetworkPolicy family routed through the live-convergence helper:- `ui-ingress`, `workload-ingress` and `workload-egress`: correctly configured UI→APIand API→sibling traffic, unlabelled negative controls, forced live-spec drift, a real- parent recipe reconcile, repair, and a post-repair resourceVersion stability window;+ parent recipe reconcile, repair, a terminating-policy race that self-heals through the+ scheduled retry without a second parent event, and a post-repair no-op witness plus+ resourceVersion stability window;- `internal-dependency`: inferred source→backend traffic, an unlabelled denied pod,two-sided policy repair and no-churn;- `oauth-broker-egress`: opted-in workload→Control API reachability, an unlabelled@@ -368,10 +374,13 @@ NetworkPolicy family routed through the live-convergence helper:- `webhook-gateway`: all three policies, a signed public webhook business signal,spec drift, stale/missing owner repair, route recovery and no-churn.-The public CI runner cannot provide this Calico-backed cluster. It therefore runs the-fail-loud composition/syntax contract, while the real dataplane result is recorded by-the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a missing family-suite, incomplete cleanup or zero executed suites are failures, never skips.+Every no-churn assertion requires both a policy/family-bound structured WRC no-op log and+an unchanged `resourceVersion`; observing only the generic parent-reconcile log is not+sufficient. The public CI runner cannot provide this Calico-backed cluster. It therefore+runs the fail-loud composition/syntax contract, while the real dataplane result is+recorded by the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a+missing family suite, incomplete cleanup or zero executed suites are failures, never+skips.### Unit tests (vitest + @testing-library/react)diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.shindex f94fe666b..9cc66bd83 100644--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh@@ -4,6 +4,8 @@set -euo pipefail+WRC_NP_LAST_RECONCILE_SINCE_TIME=""+wrc_np_spec_hash() {local namespace=$1 name=$2kctl get networkpolicy "$name" -n "$namespace" -o json |@@ -15,6 +17,11 @@ wrc_np_resource_version() {kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.resourceVersion}'}+wrc_np_uid() {+ local namespace=$1 name=$2+ kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.uid}'+}+wrc_wait_for_np() {local namespace=$1 name=$2 timeout=${3:-120} elapsed=0while [ "$elapsed" -lt "$timeout" ]; do@@ -42,6 +49,28 @@ wrc_wait_for_np_spec_hash() {return 1}+wrc_wait_for_np_recreated() {+ local namespace=$1 name=$2 previous_uid=$3 expected_hash=$4 timeout=${5:-120} elapsed=0+ local current_uid deletion_timestamp current_hash+ while [ "$elapsed" -lt "$timeout" ]; do+ current_uid="$(wrc_np_uid "$namespace" "$name" 2>/dev/null || true)"+ deletion_timestamp="$(+ kctl get networkpolicy "$name" -n "$namespace" \+ -o jsonpath='{.metadata.deletionTimestamp}' 2>/dev/null || true+ )"+ current_hash="$(wrc_np_spec_hash "$namespace" "$name" 2>/dev/null || true)"+ if [ -n "$current_uid" ] && [ "$current_uid" != "$previous_uid" ] &&+ [ -z "$deletion_timestamp" ] && [ "$current_hash" = "$expected_hash" ]; then+ ok "NetworkPolicy ${namespace}/${name} was recreated by the scheduled retry"+ return 0+ fi+ sleep "$POLL_INTERVAL"+ elapsed=$((elapsed + POLL_INTERVAL))+ done+ fail "NetworkPolicy ${namespace}/${name} was not recreated after its terminating race"+ return 1+}+wrc_inject_selector_drift() {local namespace=$1 name=$2kctl patch networkpolicy "$name" -n "$namespace" --type=merge \@@ -71,6 +100,7 @@ wrc_trigger_recipe_reconcile() {since_time="$(python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))')"+ WRC_NP_LAST_RECONCILE_SINCE_TIME="$since_time"kctl patch workflowrecipe "$name" -n "$namespace" --type=merge -p "$patch" >/dev/nullafter_generation="$(@@ -95,6 +125,49 @@ wrc_trigger_recipe_reconcile() {return 1}+wrc_wait_for_np_noop_witness() {+ local namespace=$1 name=$2 family=$3 mode=${4:-apply} timeout=${5:-120} elapsed=0+ local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}+ local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}+ local message='network policy unchanged; skipping update'++ [ -n "$WRC_NP_LAST_RECONCILE_SINCE_TIME" ] || {+ fail "No parent reconcile timestamp is available for NetworkPolicy ${namespace}/${name}"+ return 1+ }+ if [ "$mode" = "workload-egress-prefilter" ]; then+ message='network policy egress set unchanged; skipping live apply'+ elif [ "$mode" != "apply" ]; then+ fail "Unknown NetworkPolicy no-op witness mode: ${mode}"+ return 1+ fi++ while [ "$elapsed" -lt "$timeout" ]; do+ if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \+ --since-time="$WRC_NP_LAST_RECONCILE_SINCE_TIME" 2>/dev/null |+ jq -Rse --arg message "$message" --arg policy "$name" --arg namespace "$namespace" \+ --arg family "$family" '+ split("\n")+ | map(try fromjson catch null)+ | any(+ . != null and+ .msg == $message and+ .policy == $policy and+ .namespace == $namespace and+ .family == $family+ )+ ' >/dev/null; then+ ok "NetworkPolicy ${namespace}/${name} emitted its ${family} no-op witness"+ return 0+ fi+ sleep "$POLL_INTERVAL"+ elapsed=$((elapsed + POLL_INTERVAL))+ done++ fail "NetworkPolicy ${namespace}/${name} emitted no ${family} no-op witness after the parent reconcile"+ return 1+}+wrc_assert_np_stable() {local namespace=$1 name=$2 settle_seconds=${3:-20}local before afterdiff --git a/scripts/e2e/e2e-sandbox-ui-oauth.sh b/scripts/e2e/e2e-sandbox-ui-oauth.shindex f033571b8..d0a4cfeea 100755--- a/scripts/e2e/e2e-sandbox-ui-oauth.sh+++ b/scripts/e2e/e2e-sandbox-ui-oauth.sh@@ -44,8 +44,10 @@ BACKGROUND_PROBE_POD="e2e-wrc-oauth-unlabelled"STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"TIMEOUT_RECIPE_ACTIVE="${TIMEOUT_RECIPE_ACTIVE:-180}"UI_DEPLOYMENT=""+BACKGROUND_DEPLOYMENT=""UI_PORT_FWD_PORT="${E2E_OAUTH_UI_PORT:-18096}"UI_PORT_FWD_PID=""+CREATED=0wait_for_recipe_phase() {local name=$1 ns=$2 want=$3 timeout=$4 elapsed=0@@ -112,60 +114,86 @@ YAML}cleanup() {+ local cleanup_status=0header "Cleanup"if [ -n "$UI_PORT_FWD_PID" ]; then- kill "$UI_PORT_FWD_PID" 2>/dev/null || true+ if kill -0 "$UI_PORT_FWD_PID" 2>/dev/null; then+ kill "$UI_PORT_FWD_PID" 2>/dev/null || cleanup_status=1+ fiwait "$UI_PORT_FWD_PID" 2>/dev/null || trueUI_PORT_FWD_PID=""fi- kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false 2>/dev/null || true- kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false 2>/dev/null || true+ kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found \+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_status=1+ kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found --wait=false 2>/dev/null || true+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1+ wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" \+ "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1+ wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" \+ "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1# WRC GC follows ownerReferences inside the recipe namespace, but UI# workloads are reconciled cross-namespace (sandbox-recipes →# sandbox-ui) so K8s GC can't follow the link. Force-delete leftovers.if [ -n "$UI_DEPLOYMENT" ]; then- kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" --ignore-not-found 2>/dev/null || true- kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" --ignore-not-found 2>/dev/null || true+ kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1fikctl delete deployment,service -n "$SANDBOX_UI_NS" -l "clerum.io/recipe=${RECIPE_NAME}" \- --ignore-not-found 2>/dev/null || true+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl delete deployment,service,networkpolicy -n "$WORKFLOW_RECIPE_NS" \+ -l "clerum.io/recipe=${BACKGROUND_RECIPE_NAME}" --ignore-not-found \+ >/dev/null 2>&1 || cleanup_status=1kctl delete networkpolicy "ui-egress-${RECIPE_NAME}" -n "$SANDBOX_UI_NS" \- --ignore-not-found 2>/dev/null || true- kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found 2>/dev/null || true+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1kctl delete secret "$OAUTH_SECRET_NAME" -n "$WORKFLOW_RECIPE_NS" \- --ignore-not-found 2>/dev/null || true+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1++ return "$cleanup_status"}on_exit() {- local status=$?- if [ -n "$UI_PORT_FWD_PID" ]; then- kill "$UI_PORT_FWD_PID" 2>/dev/null || true- wait "$UI_PORT_FWD_PID" 2>/dev/null || true- UI_PORT_FWD_PID=""- fi+ local status=$? cleanup_status=0+ trap - EXIT INT TERMif [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; thenwarn "E2E_KEEP_RESOURCES=1; preserving OAuth fixtures for inspection"- return "$status"+ exit "$status"fi- cleanup- return "$status"+ if [ "$CREATED" = "1" ]; then+ cleanup || cleanup_status=$?+ fi+ if [ "$cleanup_status" -ne 0 ]; then+ if [ "$status" -eq 0 ]; then+ fail "OAuth E2E cleanup left resources or processes behind"+ status=1+ else+ warn "OAuth E2E cleanup also failed while preserving the original test failure"+ fi+ fi+ exit "$status"}if [[ "${1:-}" == "--cleanup-only" ]]; thencleanup- exit 0+ exit $?fitrap on_exit EXIT+trap 'exit 130' INT+trap 'exit 143' TERM# ─── Phase 0: Prerequisites ──────────────────────────────────────────check_prerequisites# ─── Phase 1: Clean Slate ────────────────────────────────────────────header "Phase 1 — Clean Slate"-cleanup+cleanup >/dev/null 2>&1 || true+CREATED=1# ─── Phase 2: Create dummy OAuth credentials Secret ──────────────────header "Phase 2 — Create dummy slack-oauth-creds Secret"@@ -237,14 +265,14 @@ if ! wait_for_recipe_phase "$BACKGROUND_RECIPE_NAME" "$WORKFLOW_RECIPE_NS" "actiexit 1fi-background_deployment="$(+BACKGROUND_DEPLOYMENT="$(wait_for_workload_instance "$BACKGROUND_RECIPE_NAME" "$BACKGROUND_WORKLOAD_ID" 120)"-[ -n "$background_deployment" ] || {+[ -n "$BACKGROUND_DEPLOYMENT" ] || {fail "Background OAuth workload instance was not assigned"exit 1}-wait_for_deployment "$WORKFLOW_RECIPE_NS" "$background_deployment" "$TIMEOUT_POD"+wait_for_deployment "$WORKFLOW_RECIPE_NS" "$BACKGROUND_DEPLOYMENT" "$TIMEOUT_POD"wrc_wait_for_np "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" 120control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.spec.clusterIP}')"@@ -253,7 +281,7 @@ control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.sexit 1}-response="$(http_response_from_target "deploy/${background_deployment}" "$control_api_ip" 8090)"+response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; thenok "Opted-in background workload reaches the OAuth broker network boundary"else@@ -273,7 +301,7 @@ oauth_policy_hash="$(wrc_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_wrc_inject_selector_drift "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME"wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120wrc_wait_for_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$oauth_policy_hash" 120-response="$(http_response_from_target "deploy/${background_deployment}" "$control_api_ip" 8090)"+response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; thenok "OAuth broker route recovered after live NetworkPolicy repair"else@@ -281,6 +309,7 @@ elseexit 1fiwrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120+wrc_wait_for_np_noop_witness "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress apply 120wrc_assert_np_stable "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$STABILITY_SECONDS"# ─── Phase 5: spec.oauthClients[] survives the round-trip ────────────diff --git a/scripts/e2e/e2e-webhooks-basic.sh b/scripts/e2e/e2e-webhooks-basic.shindex 08e9ff3d4..bb736c2c9 100755--- a/scripts/e2e/e2e-webhooks-basic.sh+++ b/scripts/e2e/e2e-webhooks-basic.sh@@ -54,6 +54,8 @@ HANDLER_INGRESS_POLICY="allow-gateway-ingress-to-handler-wf-${RECIPE_NAME}"PORT_FWD_PORT=18095PORT_FWD_PID=""+HANDLER_DEPLOYMENT=""+CREATED=0wait_for_workload_instance() {local workload_id=$1 timeout=${2:-120} elapsed=0 instance@@ -87,22 +89,75 @@ wait_for_recipe_active() {return 1}-# shellcheck disable=SC2329cleanup() {- local rc=$?- set +e- [ -n "$PORT_FWD_PID" ] && kill "$PORT_FWD_PID" 2>/dev/null- kctl -n "$GATEWAY_NS" delete -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" --ignore-not-found --wait=false >/dev/null 2>&1- kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found >/dev/null 2>&1- exit "$rc"+ local cleanup_status=0+ if [ -n "$PORT_FWD_PID" ]; then+ if kill -0 "$PORT_FWD_PID" 2>/dev/null; then+ kill "$PORT_FWD_PID" 2>/dev/null || cleanup_status=1+ fi+ wait "$PORT_FWD_PID" 2>/dev/null || true+ PORT_FWD_PID=""+ fi+ kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" \+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1+ wait_for_workflowrecipe_deleted "$GATEWAY_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \+ >/dev/null 2>&1 || cleanup_status=1+ if [ -n "$HANDLER_DEPLOYMENT" ]; then+ kctl -n "$GATEWAY_NS" delete deployment "$HANDLER_DEPLOYMENT" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete service "$HANDLER_DEPLOYMENT" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ fi+ kctl -n "$GATEWAY_NS" delete deployment,service -l "clerum.io/recipe=${RECIPE_NAME}" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete service "$GATEWAY_NAME" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete configmap "wf-${RECIPE_NAME}-webhook-gateway-config" \+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete networkpolicy "$PROXY_INGRESS_POLICY" \+ "$HANDLER_EGRESS_POLICY" "$HANDLER_INGRESS_POLICY" --ignore-not-found \+ >/dev/null 2>&1 || cleanup_status=1+ kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found \+ >/dev/null 2>&1 || cleanup_status=1+ return "$cleanup_status"+}++on_exit() {+ local status=$? cleanup_status=0+ trap - EXIT INT TERM+ if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then+ warn "E2E_KEEP_RESOURCES=1; preserving webhook fixtures for inspection"+ exit "$status"+ fi+ if [ "$CREATED" = "1" ]; then+ cleanup || cleanup_status=$?+ fi+ if [ "$cleanup_status" -ne 0 ]; then+ if [ "$status" -eq 0 ]; then+ fail "Webhook E2E cleanup left resources or processes behind"+ status=1+ else+ warn "Webhook E2E cleanup also failed while preserving the original test failure"+ fi+ fi+ exit "$status"}-trap cleanup EXIT INT TERM++if [ "${1:-}" = "--cleanup-only" ]; then+ cleanup+ exit $?+fi++trap on_exit EXIT+trap 'exit 130' INT+trap 'exit 143' TERM# ─── Phase 1: clean slate ───────────────────────────────────────────header "Phase 1 — clean slate"-kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || true-kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || true-kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found >/dev/null 2>&1 || true+cleanup >/dev/null 2>&1 || true+CREATED=1log "leftover state cleared"# ─── Phase 2: create signing-secret ─────────────────────────────────@@ -329,6 +384,9 @@ proxy_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")"handler_egress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")"handler_ingress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")"wrc_trigger_recipe_reconcile "$GATEWAY_NS" "$RECIPE_NAME" 120+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply 120+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply 120+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway apply 120sleep "$STABILITY_SECONDS"[ "$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")" = "$proxy_rv" ] || fail "${PROXY_INGRESS_POLICY} churned"[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")" = "$handler_egress_rv" ] || fail "${HANDLER_EGRESS_POLICY} churned"diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.shindex 03e97b99c..b7f186610 100755--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh@@ -304,6 +304,8 @@ printf "%s" "$repaired_output" | grep -Fq "issue485-ok" && \egress_rv="$(wrc_np_resource_version "$egress_ns" "$egress_name")"ingress_rv="$(wrc_np_resource_version "$ingress_ns" "$ingress_name")"wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120+wrc_wait_for_np_noop_witness "$egress_ns" "$egress_name" internal-dependency apply 120+wrc_wait_for_np_noop_witness "$ingress_ns" "$ingress_name" internal-dependency apply 120sleep "$STABILITY_SECONDS"[ "$(wrc_np_resource_version "$egress_ns" "$egress_name")" = "$egress_rv" ] || {fail "${egress_ref} churned after convergence"diff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.shindex f032362bc..1f6a3c46f 100755--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh@@ -24,6 +24,7 @@ STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"ROGUE_UI_POD="${RECIPE_NAME}-rogue-ui"ROGUE_WORKLOAD_POD="${RECIPE_NAME}-rogue-wl"CREATED=0+HELD_POLICY_FINALIZER=0UI_DEPLOYMENT=""API_DEPLOYMENT=""@@ -37,6 +38,12 @@ cleanup() {local status=0 namespacekctl delete pod "$ROGUE_UI_POD" -n "$SANDBOX_UI_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1kctl delete pod "$ROGUE_WORKLOAD_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1+ if [ "$HELD_POLICY_FINALIZER" = "1" ] &&+ kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" >/dev/null 2>&1; then+ kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \+ -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || status=1+ HELD_POLICY_FINALIZER=0+ fikctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" >/dev/null 2>&1 || status=1for namespace in "$SANDBOX_NS" "$SANDBOX_UI_NS"; do@@ -89,6 +96,21 @@ wait_for_recipe_active() {return 1}+wait_for_recipe_message() {+ local expected=$1 timeout=${2:-120} elapsed=0 message+ while [ "$elapsed" -lt "$timeout" ]; do+ message="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \+ -o jsonpath='{.status.message}' 2>/dev/null || true)"+ if [[ "$message" == *"$expected"* ]]; then+ return 0+ fi+ sleep "$POLL_INTERVAL"+ elapsed=$((elapsed + POLL_INTERVAL))+ done+ fail "WorkflowRecipe did not report retryable state containing: ${expected}"+ return 1+}+assert_http_body() {local description=$1 namespace=$2 deployment=$3 host=$4 port=$5 expected=$6 body# shellcheck disable=SC2016@@ -273,11 +295,34 @@ ok "WRC repaired live spec drift for all three families"assert_http_body "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"assert_http_body "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"-header "Phase 5 — steady-state no-churn"+header "Phase 5 — terminating race self-heals without another parent event"+terminating_uid="$(wrc_np_uid "$SANDBOX_NS" "$WL_INGRESS_POLICY")"+terminating_generation="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')"+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \+ -p '{"metadata":{"finalizers":["e2e.evenfire.ai/hold-deletion"]}}' >/dev/null+HELD_POLICY_FINALIZER=1+kctl delete networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --wait=false >/dev/null+wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120+wait_for_recipe_message 'is terminating; retrying after deletion' 120+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \+ -p '{"metadata":{"finalizers":[]}}' >/dev/null+HELD_POLICY_FINALIZER=0+wrc_wait_for_np_recreated "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$terminating_uid" "$wl_ingress_hash" 120+[ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')" = "$((terminating_generation + 1))" ] || {+ fail "NetworkPolicy recovery depended on an unexpected second parent spec event"+ exit 1+}+wait_for_recipe_active 120+assert_http_body "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"++header "Phase 6 — steady-state no-churn"wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress apply 120+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter 120+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingress apply 120assert_all_stable-header "Phase 6 — cleanup"+header "Phase 7 — cleanup"cleanupCREATED=0for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; dodiff --git a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.shindex 95c5e9f48..7a6c11298 100755--- a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh+++ b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh@@ -49,6 +49,16 @@ elsefail "reconcile trigger does not prove a parent recipe reconcile"fi+if grep -Fq 'network policy unchanged; skipping update' "$HELPER" &&+ grep -Fq 'network policy egress set unchanged; skipping live apply' "$HELPER" &&+ grep -Fq '.policy == $policy' "$HELPER" &&+ grep -Fq '.namespace == $namespace' "$HELPER" &&+ grep -Fq '.family == $family' "$HELPER"; then+ pass "no-churn witness binds structured WRC output to one policy and family"+else+ fail "no-churn witness can pass without proving the named policy reached its no-op path"+fi+for entry in \"$ROUTES:service-route" \"$INTDEP:internal-dependency" \@@ -64,14 +74,46 @@ for entry in \fidone+for entry in \+ "$ROUTES:3:service-route" \+ "$INTDEP:2:internal-dependency" \+ "$OAUTH:1:OAuth broker" \+ "$WEBHOOK:3:webhook"; do+ script=${entry%%:*}+ remainder=${entry#*:}+ minimum=${remainder%%:*}+ label=${remainder#*:}+ witness_count=$(grep -Fc 'wrc_wait_for_np_noop_witness' "$script" || true)+ if [ "$witness_count" -ge "$minimum" ]; then+ pass "${label} E2E requires a no-op witness for every in-scope policy"+ else+ fail "${label} E2E has ${witness_count}/${minimum} required no-op witnesses"+ fi+done++for entry in "$OAUTH:OAuth broker" "$WEBHOOK:webhook"; do+ script=${entry%%:*}+ label=${entry#*:}+ if grep -Fq 'cleanup_status=1' "$script" &&+ grep -Fq 'cleanup also failed' "$script" &&+ grep -Fq 'status=1' "$script"; then+ pass "${label} E2E makes cleanup failure fail the aggregate"+ else+ fail "${label} E2E cleanup can be silently ignored"+ fi+done+for family in UI_INGRESS_POLICY WL_EGRESS_POLICY WL_INGRESS_POLICY; dogrep -Fq "$family" "$ROUTES" || fail "service-route E2E omits $family"doneif grep -Fq 'correctly configured positive routes' "$ROUTES" &&grep -Fq 'negative controls' "$ROUTES" &&grep -Fq 'live drift repair' "$ROUTES" &&+ grep -Fq 'terminating race self-heals' "$ROUTES" &&+ grep -Fq 'wrc_wait_for_np_recreated' "$ROUTES" &&+ grep -Fq 'unexpected second parent spec event' "$ROUTES" &&grep -Fq 'steady-state no-churn' "$ROUTES"; then- pass "UI/workload route E2E proves positive, negative, repair, and no-churn signals"+ pass "UI/workload route E2E proves positive, negative, repair, scheduled retry, and no-churn signals"elsefail "UI/workload route E2E is missing a required signal"fidiff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.jsonindex 543d4a0a6..a099d73c4 100644--- a/workflow-recipes/package-lock.json+++ b/workflow-recipes/package-lock.json@@ -1,12 +1,12 @@{"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","license": "SEE LICENSE IN LICENSE","dependencies": {"@aws-sdk/client-s3": "^3.1009.0",diff --git a/workflow-recipes/package.json b/workflow-recipes/package.jsonindex e2b3242cc..474d7ac77 100644--- a/workflow-recipes/package.json+++ b/workflow-recipes/package.json@@ -1,6 +1,6 @@{"name": "workload-recipes-controller",- "version": "0.1.344",+ "version": "0.1.345","description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface","main": "dist/main.js","engines": {diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.tsindex 6daab5089..62f474273 100644--- a/workflow-recipes/src/k8sClient.test.ts+++ b/workflow-recipes/src/k8sClient.test.ts@@ -1397,6 +1397,75 @@ describe('workload status refresh loop helpers', () => {expect(reconcile).not.toHaveBeenCalled()})+ it('retries a transient degraded result even when the first pass patches status', async () => {+ vi.useFakeTimers()+ try {+ const recipe = makeWorkloadRecipe({+ metadata: { name: 'policy-race', namespace: 'sandbox-recipes', generation: 1 },+ status: { phase: 'candidate' },+ })+ const reconcile = vi+ .fn()+ .mockResolvedValueOnce({+ phase: 'degraded',+ message: 'NetworkPolicy changed during replace',+ workloadStatuses: [],+ requeueAfterMs: 5_000,+ })+ .mockResolvedValueOnce({+ phase: 'active',+ message: 'All workloads deployed',+ workloadStatuses: [],+ })+ const patchStatus = vi.fn().mockResolvedValue(undefined)+ const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())++ type InternalWatcher = {+ recipes: Map<string, WorkflowRecipeCRD>+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>+ eventQueue: { enqueue: typeof enqueue }+ stopped: boolean+ traceReporter: null+ dbRunProcessor: null+ reconciler: {+ reconcile: typeof reconcile+ isRecipeStillActive: ReturnType<typeof vi.fn>+ ensureFinalizer: ReturnType<typeof vi.fn>+ patchStatus: typeof patchStatus+ }+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>+ }++ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher+ internal.recipes = new Map([[recipe.metadata.name, recipe]])+ internal.transientRetries = new Map()+ internal.eventQueue = { enqueue }+ internal.stopped = false+ internal.traceReporter = null+ internal.dbRunProcessor = null+ internal.reconciler = {+ reconcile,+ isRecipeStillActive: vi.fn().mockResolvedValue(true),+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),+ patchStatus,+ }++ await internal.handleRecipeEvent('ADDED', recipe)++ expect(patchStatus).toHaveBeenCalledTimes(1)+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(true)++ await vi.advanceTimersByTimeAsync(5_000)+ await Promise.resolve()++ expect(enqueue).toHaveBeenCalledWith(recipe.metadata.name, expect.any(Function))+ expect(reconcile).toHaveBeenCalledTimes(2)+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(false)+ } finally {+ vi.useRealTimers()+ }+ })+it('reconciles when Codex connection-ref changes without a generation bump', async () => {const cached = makeWorkloadRecipe({metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.tsindex 27e5d7c40..7980d176b 100644--- a/workflow-recipes/src/reconciler/reconciler.test.ts+++ b/workflow-recipes/src/reconciler/reconciler.test.ts@@ -2280,6 +2280,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('degraded')expect(result.message).toMatch(/is terminating; retrying after deletion/)+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()})@@ -2322,6 +2323,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('degraded')expect(result.message).toMatch(/disappeared during replace/)+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)})it.each([{ code: 500 }, new Error('network timeout')])(@@ -2396,6 +2398,7 @@ describe('WorkflowRecipeReconciler', () => {expect(result.phase).toBe('degraded')expect(result.workflowPhase).toBeUndefined()expect(result.message).toBe('Workflow workload infrastructure temporarily unavailable (500)')+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)expect(workflowReconcile).not.toHaveBeenCalled()})@@ -6345,6 +6348,7 @@ describe('WorkflowRecipeReconciler', () => {// ownership backstop (workflowNeedsOwnershipBackstop) retries the revocation.expect(r.phase).toBe('degraded')expect(r.message).toMatch(/teardown of denied workflow workload .* failed/)+ expect(r.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)// The workflow infrastructure reconcile must NOT have run — we degraded first.expect(workflowReconcile).not.toHaveBeenCalled()@@ -6412,6 +6416,7 @@ describe('WorkflowRecipeReconciler', () => {// throwOnError makes this red (safeDelete swallows → workflow proceeds, phase 'deploying').expect(r.phase).toBe('degraded')expect(r.message).toMatch(/teardown of denied workflow workload .* failed/)+ expect(r.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)expect(workflowReconcile).not.toHaveBeenCalled()// Restore the shared delete mock (the suite clears calls, not mock implementations).diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.tsindex 20df497a0..bb72bea0e 100644--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts@@ -1938,15 +1938,18 @@ export class WorkflowRecipeReconciler {error instanceof RetryableReconcileError? message: `Workflow workload infrastructure temporarily unavailable (${safe.errorCode ?? safe.errorName})`- // Transient infra failure (e.g. DNS SERVFAIL on egress). Degrade- // instead of failing so the next reconcile retries before the- // workflow runs — don't mark workflowPhase failed.+ // Transient infra failure (e.g. DNS SERVFAIL or a NetworkPolicy+ // optimistic-concurrency race). Degrade instead of failing and+ // schedule a deterministic retry before the workflow runs — a+ // status patch alone only emits a status-only MODIFIED event and+ // cannot be relied on to enter the full reconcile path again.return {phase: 'degraded' as RecipePhase,message: retryableMessage,workloadStatuses: [],internalDependencyConditions,workloadConditions,+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,}}return {@@ -2704,8 +2707,8 @@ export class WorkflowRecipeReconciler {//// (1) RetryableReconcileError — explicitly thrown for a transient step// (e.g. DNS SERVFAIL/timeout on egress resolution). Degrade so the- // periodic reconcile retries and the recipe self-heals once the- // dependency recovers. egressResolutionError() raises this ONLY when+ // watcher schedules a deterministic retry and the recipe self-heals+ // once the dependency recovers. egressResolutionError() raises this ONLY when// every failure was transient; a permanent failure yields a plain// Error that must stay terminal (fail-closed) below.if (error instanceof RetryableReconcileError) {@@ -2716,6 +2719,7 @@ export class WorkflowRecipeReconciler {phase: 'degraded',message: error.message,workloadStatuses: [],+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,}}// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,