Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(wrc): requeue transient NetworkPolicy races

Alfredo Lopez8968b58499fd
diff --git a/docs/deploy/workflow-recipes-guide.md b/docs/deploy/workflow-recipes-guide.md
index 3462f707a..0a123f28d 100644
--- a/docs/deploy/workflow-recipes-guide.md
+++ b/docs/deploy/workflow-recipes-guide.md
@@ -334,6 +334,10 @@ it cannot hide later drift.
owner forces one optimistic-concurrency replace using the `resourceVersion` from the
same validated read on the next parent recipe reconcile. NetworkPolicy-only changes do
not emit a WorkflowRecipe event. The following reconcile is read-only.
+- **Retryable races schedule their own recovery.** A terminating object, create conflict
+ whose winner disappears, missing live `resourceVersion`, replace 404/409 or transient
+ API failure re-enqueues the parent with bounded backoff. A status-only update is not
+ treated as the retry trigger.
- **Other spec-hash-managed objects retain their existing drift contract.** For example,
a manually scaled recipe Deployment is not reverted until the desired manifest changes;
its replica-health drift remains visible in recipe status.
@@ -360,7 +364,9 @@ NetworkPolicy family routed through the live-convergence helper:
- `ui-ingress`, `workload-ingress` and `workload-egress`: correctly configured UI→API
and API→sibling traffic, unlabelled negative controls, forced live-spec drift, a real
- parent recipe reconcile, repair, and a post-repair resourceVersion stability window;
+ parent recipe reconcile, repair, a terminating-policy race that self-heals through the
+ scheduled retry without a second parent event, and a post-repair no-op witness plus
+ resourceVersion stability window;
- `internal-dependency`: inferred source→backend traffic, an unlabelled denied pod,
two-sided policy repair and no-churn;
- `oauth-broker-egress`: opted-in workload→Control API reachability, an unlabelled
@@ -368,10 +374,13 @@ NetworkPolicy family routed through the live-convergence helper:
- `webhook-gateway`: all three policies, a signed public webhook business signal,
spec drift, stale/missing owner repair, route recovery and no-churn.
-The public CI runner cannot provide this Calico-backed cluster. It therefore runs the
-fail-loud composition/syntax contract, while the real dataplane result is recorded by
-the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a missing family
-suite, incomplete cleanup or zero executed suites are failures, never skips.
+Every no-churn assertion requires both a policy/family-bound structured WRC no-op log and
+an unchanged `resourceVersion`; observing only the generic parent-reconcile log is not
+sufficient. The public CI runner cannot provide this Calico-backed cluster. It therefore
+runs the fail-loud composition/syntax contract, while the real dataplane result is
+recorded by the exact-head local T2/pre-merge evidence lane. Missing prerequisites, a
+missing family suite, incomplete cleanup or zero executed suites are failures, never
+skips.
### Unit tests (vitest + @testing-library/react)
diff --git a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
index f94fe666b..9cc66bd83 100644
--- a/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
+++ b/scripts/e2e/_lib/wrc-networkpolicy-convergence.sh
@@ -4,6 +4,8 @@
set -euo pipefail
+WRC_NP_LAST_RECONCILE_SINCE_TIME=""
+
wrc_np_spec_hash() {
local namespace=$1 name=$2
kctl get networkpolicy "$name" -n "$namespace" -o json |
@@ -15,6 +17,11 @@ wrc_np_resource_version() {
kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.resourceVersion}'
}
+wrc_np_uid() {
+ local namespace=$1 name=$2
+ kctl get networkpolicy "$name" -n "$namespace" -o jsonpath='{.metadata.uid}'
+}
+
wrc_wait_for_np() {
local namespace=$1 name=$2 timeout=${3:-120} elapsed=0
while [ "$elapsed" -lt "$timeout" ]; do
@@ -42,6 +49,28 @@ wrc_wait_for_np_spec_hash() {
return 1
}
+wrc_wait_for_np_recreated() {
+ local namespace=$1 name=$2 previous_uid=$3 expected_hash=$4 timeout=${5:-120} elapsed=0
+ local current_uid deletion_timestamp current_hash
+ while [ "$elapsed" -lt "$timeout" ]; do
+ current_uid="$(wrc_np_uid "$namespace" "$name" 2>/dev/null || true)"
+ deletion_timestamp="$(
+ kctl get networkpolicy "$name" -n "$namespace" \
+ -o jsonpath='{.metadata.deletionTimestamp}' 2>/dev/null || true
+ )"
+ current_hash="$(wrc_np_spec_hash "$namespace" "$name" 2>/dev/null || true)"
+ if [ -n "$current_uid" ] && [ "$current_uid" != "$previous_uid" ] &&
+ [ -z "$deletion_timestamp" ] && [ "$current_hash" = "$expected_hash" ]; then
+ ok "NetworkPolicy ${namespace}/${name} was recreated by the scheduled retry"
+ return 0
+ fi
+ sleep "$POLL_INTERVAL"
+ elapsed=$((elapsed + POLL_INTERVAL))
+ done
+ fail "NetworkPolicy ${namespace}/${name} was not recreated after its terminating race"
+ return 1
+}
+
wrc_inject_selector_drift() {
local namespace=$1 name=$2
kctl patch networkpolicy "$name" -n "$namespace" --type=merge \
@@ -71,6 +100,7 @@ wrc_trigger_recipe_reconcile() {
since_time="$(
python3 -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"))'
)"
+ WRC_NP_LAST_RECONCILE_SINCE_TIME="$since_time"
kctl patch workflowrecipe "$name" -n "$namespace" --type=merge -p "$patch" >/dev/null
after_generation="$(
@@ -95,6 +125,49 @@ wrc_trigger_recipe_reconcile() {
return 1
}
+wrc_wait_for_np_noop_witness() {
+ local namespace=$1 name=$2 family=$3 mode=${4:-apply} timeout=${5:-120} elapsed=0
+ local controller_namespace=${WRC_CONTROLLER_NAMESPACE:-control-plane}
+ local controller_deployment=${WRC_CONTROLLER_DEPLOYMENT:-workflow-recipes}
+ local message='network policy unchanged; skipping update'
+
+ [ -n "$WRC_NP_LAST_RECONCILE_SINCE_TIME" ] || {
+ fail "No parent reconcile timestamp is available for NetworkPolicy ${namespace}/${name}"
+ return 1
+ }
+ if [ "$mode" = "workload-egress-prefilter" ]; then
+ message='network policy egress set unchanged; skipping live apply'
+ elif [ "$mode" != "apply" ]; then
+ fail "Unknown NetworkPolicy no-op witness mode: ${mode}"
+ return 1
+ fi
+
+ while [ "$elapsed" -lt "$timeout" ]; do
+ if kctl logs -n "$controller_namespace" "deployment/${controller_deployment}" \
+ --since-time="$WRC_NP_LAST_RECONCILE_SINCE_TIME" 2>/dev/null |
+ jq -Rse --arg message "$message" --arg policy "$name" --arg namespace "$namespace" \
+ --arg family "$family" '
+ split("\n")
+ | map(try fromjson catch null)
+ | any(
+ . != null and
+ .msg == $message and
+ .policy == $policy and
+ .namespace == $namespace and
+ .family == $family
+ )
+ ' >/dev/null; then
+ ok "NetworkPolicy ${namespace}/${name} emitted its ${family} no-op witness"
+ return 0
+ fi
+ sleep "$POLL_INTERVAL"
+ elapsed=$((elapsed + POLL_INTERVAL))
+ done
+
+ fail "NetworkPolicy ${namespace}/${name} emitted no ${family} no-op witness after the parent reconcile"
+ return 1
+}
+
wrc_assert_np_stable() {
local namespace=$1 name=$2 settle_seconds=${3:-20}
local before after
diff --git a/scripts/e2e/e2e-sandbox-ui-oauth.sh b/scripts/e2e/e2e-sandbox-ui-oauth.sh
index f033571b8..d0a4cfeea 100755
--- a/scripts/e2e/e2e-sandbox-ui-oauth.sh
+++ b/scripts/e2e/e2e-sandbox-ui-oauth.sh
@@ -44,8 +44,10 @@ BACKGROUND_PROBE_POD="e2e-wrc-oauth-unlabelled"
STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
TIMEOUT_RECIPE_ACTIVE="${TIMEOUT_RECIPE_ACTIVE:-180}"
UI_DEPLOYMENT=""
+BACKGROUND_DEPLOYMENT=""
UI_PORT_FWD_PORT="${E2E_OAUTH_UI_PORT:-18096}"
UI_PORT_FWD_PID=""
+CREATED=0
wait_for_recipe_phase() {
local name=$1 ns=$2 want=$3 timeout=$4 elapsed=0
@@ -112,60 +114,86 @@ YAML
}
cleanup() {
+ local cleanup_status=0
header "Cleanup"
if [ -n "$UI_PORT_FWD_PID" ]; then
- kill "$UI_PORT_FWD_PID" 2>/dev/null || true
+ if kill -0 "$UI_PORT_FWD_PID" 2>/dev/null; then
+ kill "$UI_PORT_FWD_PID" 2>/dev/null || cleanup_status=1
+ fi
wait "$UI_PORT_FWD_PID" 2>/dev/null || true
UI_PORT_FWD_PID=""
fi
- kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false 2>/dev/null || true
- kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false 2>/dev/null || true
+ kctl delete pod "$BACKGROUND_PROBE_POD" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found \
+ --wait=true --timeout=60s >/dev/null 2>&1 || cleanup_status=1
+ kctl delete workflowrecipe "$BACKGROUND_RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found --wait=false 2>/dev/null || true
+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
+ wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" \
+ "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1
+ wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" \
+ "$TIMEOUT_DELETE" >/dev/null 2>&1 || cleanup_status=1
# WRC GC follows ownerReferences inside the recipe namespace, but UI
# workloads are reconciled cross-namespace (sandbox-recipes →
# sandbox-ui) so K8s GC can't follow the link. Force-delete leftovers.
if [ -n "$UI_DEPLOYMENT" ]; then
- kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" --ignore-not-found 2>/dev/null || true
- kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" --ignore-not-found 2>/dev/null || true
+ kctl delete deployment "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl delete svc "$UI_DEPLOYMENT" -n "$SANDBOX_UI_NS" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
fi
kctl delete deployment,service -n "$SANDBOX_UI_NS" -l "clerum.io/recipe=${RECIPE_NAME}" \
- --ignore-not-found 2>/dev/null || true
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl delete deployment,service,networkpolicy -n "$WORKFLOW_RECIPE_NS" \
+ -l "clerum.io/recipe=${BACKGROUND_RECIPE_NAME}" --ignore-not-found \
+ >/dev/null 2>&1 || cleanup_status=1
kctl delete networkpolicy "ui-egress-${RECIPE_NAME}" -n "$SANDBOX_UI_NS" \
- --ignore-not-found 2>/dev/null || true
- kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found 2>/dev/null || true
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl delete networkpolicy "$BACKGROUND_POLICY_NAME" -n "$WORKFLOW_RECIPE_NS" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
kctl delete secret "$OAUTH_SECRET_NAME" -n "$WORKFLOW_RECIPE_NS" \
- --ignore-not-found 2>/dev/null || true
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+
+ return "$cleanup_status"
}
on_exit() {
- local status=$?
- if [ -n "$UI_PORT_FWD_PID" ]; then
- kill "$UI_PORT_FWD_PID" 2>/dev/null || true
- wait "$UI_PORT_FWD_PID" 2>/dev/null || true
- UI_PORT_FWD_PID=""
- fi
+ local status=$? cleanup_status=0
+ trap - EXIT INT TERM
if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then
warn "E2E_KEEP_RESOURCES=1; preserving OAuth fixtures for inspection"
- return "$status"
+ exit "$status"
fi
- cleanup
- return "$status"
+ if [ "$CREATED" = "1" ]; then
+ cleanup || cleanup_status=$?
+ fi
+ if [ "$cleanup_status" -ne 0 ]; then
+ if [ "$status" -eq 0 ]; then
+ fail "OAuth E2E cleanup left resources or processes behind"
+ status=1
+ else
+ warn "OAuth E2E cleanup also failed while preserving the original test failure"
+ fi
+ fi
+ exit "$status"
}
if [[ "${1:-}" == "--cleanup-only" ]]; then
cleanup
- exit 0
+ exit $?
fi
trap on_exit EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
# ─── Phase 0: Prerequisites ──────────────────────────────────────────
check_prerequisites
# ─── Phase 1: Clean Slate ────────────────────────────────────────────
header "Phase 1 — Clean Slate"
-cleanup
+cleanup >/dev/null 2>&1 || true
+CREATED=1
# ─── Phase 2: Create dummy OAuth credentials Secret ──────────────────
header "Phase 2 — Create dummy slack-oauth-creds Secret"
@@ -237,14 +265,14 @@ if ! wait_for_recipe_phase "$BACKGROUND_RECIPE_NAME" "$WORKFLOW_RECIPE_NS" "acti
exit 1
fi
-background_deployment="$(
+BACKGROUND_DEPLOYMENT="$(
wait_for_workload_instance "$BACKGROUND_RECIPE_NAME" "$BACKGROUND_WORKLOAD_ID" 120
)"
-[ -n "$background_deployment" ] || {
+[ -n "$BACKGROUND_DEPLOYMENT" ] || {
fail "Background OAuth workload instance was not assigned"
exit 1
}
-wait_for_deployment "$WORKFLOW_RECIPE_NS" "$background_deployment" "$TIMEOUT_POD"
+wait_for_deployment "$WORKFLOW_RECIPE_NS" "$BACKGROUND_DEPLOYMENT" "$TIMEOUT_POD"
wrc_wait_for_np "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" 120
control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.spec.clusterIP}')"
@@ -253,7 +281,7 @@ control_api_ip="$(kctl get service control-api -n "$CONTROL_NS" -o jsonpath='{.s
exit 1
}
-response="$(http_response_from_target "deploy/${background_deployment}" "$control_api_ip" 8090)"
+response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"
if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then
ok "Opted-in background workload reaches the OAuth broker network boundary"
else
@@ -273,7 +301,7 @@ oauth_policy_hash="$(wrc_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_
wrc_inject_selector_drift "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME"
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120
wrc_wait_for_np_spec_hash "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$oauth_policy_hash" 120
-response="$(http_response_from_target "deploy/${background_deployment}" "$control_api_ip" 8090)"
+response="$(http_response_from_target "deploy/${BACKGROUND_DEPLOYMENT}" "$control_api_ip" 8090)"
if printf '%s' "$response" | grep -Eq 'HTTP/1\.[01] [2345][0-9][0-9]'; then
ok "OAuth broker route recovered after live NetworkPolicy repair"
else
@@ -281,6 +309,7 @@ else
exit 1
fi
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$BACKGROUND_RECIPE_NAME" 120
+wrc_wait_for_np_noop_witness "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" oauth-broker-egress apply 120
wrc_assert_np_stable "$WORKFLOW_RECIPE_NS" "$BACKGROUND_POLICY_NAME" "$STABILITY_SECONDS"
# ─── Phase 5: spec.oauthClients[] survives the round-trip ────────────
diff --git a/scripts/e2e/e2e-webhooks-basic.sh b/scripts/e2e/e2e-webhooks-basic.sh
index 08e9ff3d4..bb736c2c9 100755
--- a/scripts/e2e/e2e-webhooks-basic.sh
+++ b/scripts/e2e/e2e-webhooks-basic.sh
@@ -54,6 +54,8 @@ HANDLER_INGRESS_POLICY="allow-gateway-ingress-to-handler-wf-${RECIPE_NAME}"
PORT_FWD_PORT=18095
PORT_FWD_PID=""
+HANDLER_DEPLOYMENT=""
+CREATED=0
wait_for_workload_instance() {
local workload_id=$1 timeout=${2:-120} elapsed=0 instance
@@ -87,22 +89,75 @@ wait_for_recipe_active() {
return 1
}
-# shellcheck disable=SC2329
cleanup() {
- local rc=$?
- set +e
- [ -n "$PORT_FWD_PID" ] && kill "$PORT_FWD_PID" 2>/dev/null
- kctl -n "$GATEWAY_NS" delete -f "${SCRIPT_DIR}/../../${RECIPE_FILE}" --ignore-not-found --wait=false >/dev/null 2>&1
- kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found >/dev/null 2>&1
- exit "$rc"
+ local cleanup_status=0
+ if [ -n "$PORT_FWD_PID" ]; then
+ if kill -0 "$PORT_FWD_PID" 2>/dev/null; then
+ kill "$PORT_FWD_PID" 2>/dev/null || cleanup_status=1
+ fi
+ wait "$PORT_FWD_PID" 2>/dev/null || true
+ PORT_FWD_PID=""
+ fi
+ kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" \
+ --ignore-not-found --wait=false >/dev/null 2>&1 || cleanup_status=1
+ wait_for_workflowrecipe_deleted "$GATEWAY_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" \
+ >/dev/null 2>&1 || cleanup_status=1
+ if [ -n "$HANDLER_DEPLOYMENT" ]; then
+ kctl -n "$GATEWAY_NS" delete deployment "$HANDLER_DEPLOYMENT" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete service "$HANDLER_DEPLOYMENT" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ fi
+ kctl -n "$GATEWAY_NS" delete deployment,service -l "clerum.io/recipe=${RECIPE_NAME}" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete service "$GATEWAY_NAME" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete configmap "wf-${RECIPE_NAME}-webhook-gateway-config" \
+ --ignore-not-found >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete networkpolicy "$PROXY_INGRESS_POLICY" \
+ "$HANDLER_EGRESS_POLICY" "$HANDLER_INGRESS_POLICY" --ignore-not-found \
+ >/dev/null 2>&1 || cleanup_status=1
+ kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found \
+ >/dev/null 2>&1 || cleanup_status=1
+ return "$cleanup_status"
+}
+
+on_exit() {
+ local status=$? cleanup_status=0
+ trap - EXIT INT TERM
+ if [ "${E2E_KEEP_RESOURCES:-0}" = "1" ]; then
+ warn "E2E_KEEP_RESOURCES=1; preserving webhook fixtures for inspection"
+ exit "$status"
+ fi
+ if [ "$CREATED" = "1" ]; then
+ cleanup || cleanup_status=$?
+ fi
+ if [ "$cleanup_status" -ne 0 ]; then
+ if [ "$status" -eq 0 ]; then
+ fail "Webhook E2E cleanup left resources or processes behind"
+ status=1
+ else
+ warn "Webhook E2E cleanup also failed while preserving the original test failure"
+ fi
+ fi
+ exit "$status"
}
-trap cleanup EXIT INT TERM
+
+if [ "${1:-}" = "--cleanup-only" ]; then
+ cleanup
+ exit $?
+fi
+
+trap on_exit EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
# ─── Phase 1: clean slate ───────────────────────────────────────────
header "Phase 1 — clean slate"
-kctl -n "$GATEWAY_NS" delete workflowrecipe "$RECIPE_NAME" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || true
-kctl -n "$GATEWAY_NS" delete deployment "$GATEWAY_NAME" --ignore-not-found --wait=true --timeout=60s >/dev/null 2>&1 || true
-kctl -n "$GATEWAY_NS" delete secret "$SECRET_NAME" --ignore-not-found >/dev/null 2>&1 || true
+cleanup >/dev/null 2>&1 || true
+CREATED=1
log "leftover state cleared"
# ─── Phase 2: create signing-secret ─────────────────────────────────
@@ -329,6 +384,9 @@ proxy_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")"
handler_egress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")"
handler_ingress_rv="$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY")"
wrc_trigger_recipe_reconcile "$GATEWAY_NS" "$RECIPE_NAME" 120
+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$PROXY_INGRESS_POLICY" webhook-gateway apply 120
+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY" webhook-gateway apply 120
+wrc_wait_for_np_noop_witness "$GATEWAY_NS" "$HANDLER_INGRESS_POLICY" webhook-gateway apply 120
sleep "$STABILITY_SECONDS"
[ "$(wrc_np_resource_version "$GATEWAY_NS" "$PROXY_INGRESS_POLICY")" = "$proxy_rv" ] || fail "${PROXY_INGRESS_POLICY} churned"
[ "$(wrc_np_resource_version "$GATEWAY_NS" "$HANDLER_EGRESS_POLICY")" = "$handler_egress_rv" ] || fail "${HANDLER_EGRESS_POLICY} churned"
diff --git a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
index 03e97b99c..b7f186610 100755
--- a/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
+++ b/scripts/e2e/e2e-wrc-internal-dependency-networkpolicy.sh
@@ -304,6 +304,8 @@ printf "%s" "$repaired_output" | grep -Fq "issue485-ok" && \
egress_rv="$(wrc_np_resource_version "$egress_ns" "$egress_name")"
ingress_rv="$(wrc_np_resource_version "$ingress_ns" "$ingress_name")"
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
+wrc_wait_for_np_noop_witness "$egress_ns" "$egress_name" internal-dependency apply 120
+wrc_wait_for_np_noop_witness "$ingress_ns" "$ingress_name" internal-dependency apply 120
sleep "$STABILITY_SECONDS"
[ "$(wrc_np_resource_version "$egress_ns" "$egress_name")" = "$egress_rv" ] || {
fail "${egress_ref} churned after convergence"
diff --git a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
index f032362bc..1f6a3c46f 100755
--- a/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
+++ b/scripts/e2e/e2e-wrc-networkpolicy-service-routes.sh
@@ -24,6 +24,7 @@ STABILITY_SECONDS="${E2E_NP_STABILITY_SECONDS:-20}"
ROGUE_UI_POD="${RECIPE_NAME}-rogue-ui"
ROGUE_WORKLOAD_POD="${RECIPE_NAME}-rogue-wl"
CREATED=0
+HELD_POLICY_FINALIZER=0
UI_DEPLOYMENT=""
API_DEPLOYMENT=""
@@ -37,6 +38,12 @@ cleanup() {
local status=0 namespace
kctl delete pod "$ROGUE_UI_POD" -n "$SANDBOX_UI_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
kctl delete pod "$ROGUE_WORKLOAD_POD" -n "$SANDBOX_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
+ if [ "$HELD_POLICY_FINALIZER" = "1" ] &&
+ kctl get networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" >/dev/null 2>&1; then
+ kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
+ -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || status=1
+ HELD_POLICY_FINALIZER=0
+ fi
kctl delete workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || status=1
wait_for_workflowrecipe_deleted "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" "$TIMEOUT_DELETE" >/dev/null 2>&1 || status=1
for namespace in "$SANDBOX_NS" "$SANDBOX_UI_NS"; do
@@ -89,6 +96,21 @@ wait_for_recipe_active() {
return 1
}
+wait_for_recipe_message() {
+ local expected=$1 timeout=${2:-120} elapsed=0 message
+ while [ "$elapsed" -lt "$timeout" ]; do
+ message="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" \
+ -o jsonpath='{.status.message}' 2>/dev/null || true)"
+ if [[ "$message" == *"$expected"* ]]; then
+ return 0
+ fi
+ sleep "$POLL_INTERVAL"
+ elapsed=$((elapsed + POLL_INTERVAL))
+ done
+ fail "WorkflowRecipe did not report retryable state containing: ${expected}"
+ return 1
+}
+
assert_http_body() {
local description=$1 namespace=$2 deployment=$3 host=$4 port=$5 expected=$6 body
# shellcheck disable=SC2016
@@ -273,11 +295,34 @@ ok "WRC repaired live spec drift for all three families"
assert_http_body "sandbox UI route recovered after NetworkPolicy repair" "$SANDBOX_UI_NS" "$UI_DEPLOYMENT" "$API_CLUSTER_IP" 8080 "api-ok"
assert_http_body "workload sibling route recovered after NetworkPolicy repair" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
-header "Phase 5 — steady-state no-churn"
+header "Phase 5 — terminating race self-heals without another parent event"
+terminating_uid="$(wrc_np_uid "$SANDBOX_NS" "$WL_INGRESS_POLICY")"
+terminating_generation="$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')"
+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
+ -p '{"metadata":{"finalizers":["e2e.evenfire.ai/hold-deletion"]}}' >/dev/null
+HELD_POLICY_FINALIZER=1
+kctl delete networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --wait=false >/dev/null
+wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
+wait_for_recipe_message 'is terminating; retrying after deletion' 120
+kctl patch networkpolicy "$WL_INGRESS_POLICY" -n "$SANDBOX_NS" --type=merge \
+ -p '{"metadata":{"finalizers":[]}}' >/dev/null
+HELD_POLICY_FINALIZER=0
+wrc_wait_for_np_recreated "$SANDBOX_NS" "$WL_INGRESS_POLICY" "$terminating_uid" "$wl_ingress_hash" 120
+[ "$(kctl get workflowrecipe "$RECIPE_NAME" -n "$WORKFLOW_RECIPE_NS" -o jsonpath='{.metadata.generation}')" = "$((terminating_generation + 1))" ] || {
+ fail "NetworkPolicy recovery depended on an unexpected second parent spec event"
+ exit 1
+}
+wait_for_recipe_active 120
+assert_http_body "workload sibling route recovered after scheduled NetworkPolicy retry" "$SANDBOX_NS" "$API_DEPLOYMENT" "$DB_CLUSTER_IP" "$DB_PORT" "db-ok"
+
+header "Phase 6 — steady-state no-churn"
wrc_trigger_recipe_reconcile "$WORKFLOW_RECIPE_NS" "$RECIPE_NAME" 120
+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$UI_INGRESS_POLICY" ui-ingress apply 120
+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_EGRESS_POLICY" workload-egress workload-egress-prefilter 120
+wrc_wait_for_np_noop_witness "$SANDBOX_NS" "$WL_INGRESS_POLICY" workload-ingress apply 120
assert_all_stable
-header "Phase 6 — cleanup"
+header "Phase 7 — cleanup"
cleanup
CREATED=0
for policy in "$UI_INGRESS_POLICY" "$WL_EGRESS_POLICY" "$WL_INGRESS_POLICY"; do
diff --git a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
index 95c5e9f48..7a6c11298 100755
--- a/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
+++ b/scripts/tests/test-wrc-networkpolicy-live-convergence-e2e.sh
@@ -49,6 +49,16 @@ else
fail "reconcile trigger does not prove a parent recipe reconcile"
fi
+if grep -Fq 'network policy unchanged; skipping update' "$HELPER" &&
+ grep -Fq 'network policy egress set unchanged; skipping live apply' "$HELPER" &&
+ grep -Fq '.policy == $policy' "$HELPER" &&
+ grep -Fq '.namespace == $namespace' "$HELPER" &&
+ grep -Fq '.family == $family' "$HELPER"; then
+ pass "no-churn witness binds structured WRC output to one policy and family"
+else
+ fail "no-churn witness can pass without proving the named policy reached its no-op path"
+fi
+
for entry in \
"$ROUTES:service-route" \
"$INTDEP:internal-dependency" \
@@ -64,14 +74,46 @@ for entry in \
fi
done
+for entry in \
+ "$ROUTES:3:service-route" \
+ "$INTDEP:2:internal-dependency" \
+ "$OAUTH:1:OAuth broker" \
+ "$WEBHOOK:3:webhook"; do
+ script=${entry%%:*}
+ remainder=${entry#*:}
+ minimum=${remainder%%:*}
+ label=${remainder#*:}
+ witness_count=$(grep -Fc 'wrc_wait_for_np_noop_witness' "$script" || true)
+ if [ "$witness_count" -ge "$minimum" ]; then
+ pass "${label} E2E requires a no-op witness for every in-scope policy"
+ else
+ fail "${label} E2E has ${witness_count}/${minimum} required no-op witnesses"
+ fi
+done
+
+for entry in "$OAUTH:OAuth broker" "$WEBHOOK:webhook"; do
+ script=${entry%%:*}
+ label=${entry#*:}
+ if grep -Fq 'cleanup_status=1' "$script" &&
+ grep -Fq 'cleanup also failed' "$script" &&
+ grep -Fq 'status=1' "$script"; then
+ pass "${label} E2E makes cleanup failure fail the aggregate"
+ else
+ fail "${label} E2E cleanup can be silently ignored"
+ fi
+done
+
for family in UI_INGRESS_POLICY WL_EGRESS_POLICY WL_INGRESS_POLICY; do
grep -Fq "$family" "$ROUTES" || fail "service-route E2E omits $family"
done
if grep -Fq 'correctly configured positive routes' "$ROUTES" &&
grep -Fq 'negative controls' "$ROUTES" &&
grep -Fq 'live drift repair' "$ROUTES" &&
+ grep -Fq 'terminating race self-heals' "$ROUTES" &&
+ grep -Fq 'wrc_wait_for_np_recreated' "$ROUTES" &&
+ grep -Fq 'unexpected second parent spec event' "$ROUTES" &&
grep -Fq 'steady-state no-churn' "$ROUTES"; then
- pass "UI/workload route E2E proves positive, negative, repair, and no-churn signals"
+ pass "UI/workload route E2E proves positive, negative, repair, scheduled retry, and no-churn signals"
else
fail "UI/workload route E2E is missing a required signal"
fi
diff --git a/workflow-recipes/package-lock.json b/workflow-recipes/package-lock.json
index 543d4a0a6..a099d73c4 100644
--- a/workflow-recipes/package-lock.json
+++ b/workflow-recipes/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"license": "SEE LICENSE IN LICENSE",
"dependencies": {
"@aws-sdk/client-s3": "^3.1009.0",
diff --git a/workflow-recipes/package.json b/workflow-recipes/package.json
index e2b3242cc..474d7ac77 100644
--- a/workflow-recipes/package.json
+++ b/workflow-recipes/package.json
@@ -1,6 +1,6 @@
{
"name": "workload-recipes-controller",
- "version": "0.1.344",
+ "version": "0.1.345",
"description": "Workload Recipes Controller (WRC) - WorkloadRecipe lifecycle management and MCP server interface",
"main": "dist/main.js",
"engines": {
diff --git a/workflow-recipes/src/k8sClient.test.ts b/workflow-recipes/src/k8sClient.test.ts
index 6daab5089..62f474273 100644
--- a/workflow-recipes/src/k8sClient.test.ts
+++ b/workflow-recipes/src/k8sClient.test.ts
@@ -1397,6 +1397,75 @@ describe('workload status refresh loop helpers', () => {
expect(reconcile).not.toHaveBeenCalled()
})
+ it('retries a transient degraded result even when the first pass patches status', async () => {
+ vi.useFakeTimers()
+ try {
+ const recipe = makeWorkloadRecipe({
+ metadata: { name: 'policy-race', namespace: 'sandbox-recipes', generation: 1 },
+ status: { phase: 'candidate' },
+ })
+ const reconcile = vi
+ .fn()
+ .mockResolvedValueOnce({
+ phase: 'degraded',
+ message: 'NetworkPolicy changed during replace',
+ workloadStatuses: [],
+ requeueAfterMs: 5_000,
+ })
+ .mockResolvedValueOnce({
+ phase: 'active',
+ message: 'All workloads deployed',
+ workloadStatuses: [],
+ })
+ const patchStatus = vi.fn().mockResolvedValue(undefined)
+ const enqueue = vi.fn((_key: string, task: () => Promise<void>) => task())
+
+ type InternalWatcher = {
+ recipes: Map<string, WorkflowRecipeCRD>
+ transientRetries: Map<string, { timer: ReturnType<typeof setTimeout>; attempts: number }>
+ eventQueue: { enqueue: typeof enqueue }
+ stopped: boolean
+ traceReporter: null
+ dbRunProcessor: null
+ reconciler: {
+ reconcile: typeof reconcile
+ isRecipeStillActive: ReturnType<typeof vi.fn>
+ ensureFinalizer: ReturnType<typeof vi.fn>
+ patchStatus: typeof patchStatus
+ }
+ handleRecipeEvent: (type: string, recipe: WorkflowRecipeCRD) => Promise<void>
+ }
+
+ const internal = Object.create(WorkflowRecipeWatcher.prototype) as InternalWatcher
+ internal.recipes = new Map([[recipe.metadata.name, recipe]])
+ internal.transientRetries = new Map()
+ internal.eventQueue = { enqueue }
+ internal.stopped = false
+ internal.traceReporter = null
+ internal.dbRunProcessor = null
+ internal.reconciler = {
+ reconcile,
+ isRecipeStillActive: vi.fn().mockResolvedValue(true),
+ ensureFinalizer: vi.fn().mockResolvedValue(undefined),
+ patchStatus,
+ }
+
+ await internal.handleRecipeEvent('ADDED', recipe)
+
+ expect(patchStatus).toHaveBeenCalledTimes(1)
+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(true)
+
+ await vi.advanceTimersByTimeAsync(5_000)
+ await Promise.resolve()
+
+ expect(enqueue).toHaveBeenCalledWith(recipe.metadata.name, expect.any(Function))
+ expect(reconcile).toHaveBeenCalledTimes(2)
+ expect(internal.transientRetries.has(recipe.metadata.name)).toBe(false)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
it('reconciles when Codex connection-ref changes without a generation bump', async () => {
const cached = makeWorkloadRecipe({
metadata: { name: 'active-recipe', namespace: 'sandbox-recipes', generation: 7 },
diff --git a/workflow-recipes/src/reconciler/reconciler.test.ts b/workflow-recipes/src/reconciler/reconciler.test.ts
index 27e5d7c40..7980d176b 100644
--- a/workflow-recipes/src/reconciler/reconciler.test.ts
+++ b/workflow-recipes/src/reconciler/reconciler.test.ts
@@ -2280,6 +2280,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('degraded')
expect(result.message).toMatch(/is terminating; retrying after deletion/)
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
expect(mockNetworkingApi.replaceNamespacedNetworkPolicy).not.toHaveBeenCalled()
})
@@ -2322,6 +2323,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('degraded')
expect(result.message).toMatch(/disappeared during replace/)
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
})
it.each([{ code: 500 }, new Error('network timeout')])(
@@ -2396,6 +2398,7 @@ describe('WorkflowRecipeReconciler', () => {
expect(result.phase).toBe('degraded')
expect(result.workflowPhase).toBeUndefined()
expect(result.message).toBe('Workflow workload infrastructure temporarily unavailable (500)')
+ expect(result.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
expect(workflowReconcile).not.toHaveBeenCalled()
})
@@ -6345,6 +6348,7 @@ describe('WorkflowRecipeReconciler', () => {
// ownership backstop (workflowNeedsOwnershipBackstop) retries the revocation.
expect(r.phase).toBe('degraded')
expect(r.message).toMatch(/teardown of denied workflow workload .* failed/)
+ expect(r.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
// The workflow infrastructure reconcile must NOT have run — we degraded first.
expect(workflowReconcile).not.toHaveBeenCalled()
@@ -6412,6 +6416,7 @@ describe('WorkflowRecipeReconciler', () => {
// throwOnError makes this red (safeDelete swallows → workflow proceeds, phase 'deploying').
expect(r.phase).toBe('degraded')
expect(r.message).toMatch(/teardown of denied workflow workload .* failed/)
+ expect(r.requeueAfterMs).toBe(TRANSIENT_REQUEUE_BASE_MS)
expect(workflowReconcile).not.toHaveBeenCalled()
// Restore the shared delete mock (the suite clears calls, not mock implementations).
diff --git a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
index 20df497a0..bb72bea0e 100644
--- a/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
+++ b/workflow-recipes/src/reconciler/workflowRecipeReconciler.ts
@@ -1938,15 +1938,18 @@ export class WorkflowRecipeReconciler {
error instanceof RetryableReconcileError
? message
: `Workflow workload infrastructure temporarily unavailable (${safe.errorCode ?? safe.errorName})`
- // Transient infra failure (e.g. DNS SERVFAIL on egress). Degrade
- // instead of failing so the next reconcile retries before the
- // workflow runs — don't mark workflowPhase failed.
+ // Transient infra failure (e.g. DNS SERVFAIL or a NetworkPolicy
+ // optimistic-concurrency race). Degrade instead of failing and
+ // schedule a deterministic retry before the workflow runs — a
+ // status patch alone only emits a status-only MODIFIED event and
+ // cannot be relied on to enter the full reconcile path again.
return {
phase: 'degraded' as RecipePhase,
message: retryableMessage,
workloadStatuses: [],
internalDependencyConditions,
workloadConditions,
+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
}
}
return {
@@ -2704,8 +2707,8 @@ export class WorkflowRecipeReconciler {
//
// (1) RetryableReconcileError — explicitly thrown for a transient step
// (e.g. DNS SERVFAIL/timeout on egress resolution). Degrade so the
- // periodic reconcile retries and the recipe self-heals once the
- // dependency recovers. egressResolutionError() raises this ONLY when
+ // watcher schedules a deterministic retry and the recipe self-heals
+ // once the dependency recovers. egressResolutionError() raises this ONLY when
// every failure was transient; a permanent failure yields a plain
// Error that must stay terminal (fail-closed) below.
if (error instanceof RetryableReconcileError) {
@@ -2716,6 +2719,7 @@ export class WorkflowRecipeReconciler {
phase: 'degraded',
message: error.message,
workloadStatuses: [],
+ requeueAfterMs: TRANSIENT_REQUEUE_BASE_MS,
}
}
// (2) A raw retryable infra error (connect ETIMEDOUT/ECONNRESET, 5xx,